00000400:00000010:0.0F:1775764330.070424:0:2340:0:(lib-cpt.c:111:cfs_cpt_table_alloc()) alloc '(cptab)': 56 at 00000000610d03b4. 00000400:00000010:0.0:1775764330.070450:0:2340:0:(lib-cpt.c:120:cfs_cpt_table_alloc()) alloc '(cptab->ctb_nodemask)': 128 at 0000000066d0d5a9. 00000400:00000010:0.0:1775764330.070454:0:2340:0:(lib-cpt.c:124:cfs_cpt_table_alloc()) alloc '(cptab->ctb_cpu2cpt)': 16 at 000000006765bc1d. 00000400:00000010:0.0:1775764330.070457:0:2340:0:(lib-cpt.c:131:cfs_cpt_table_alloc()) alloc '(cptab->ctb_node2cpt)': 4 at 00000000706d801c. 00000400:00000010:0.0:1775764330.070459:0:2340:0:(lib-cpt.c:138:cfs_cpt_table_alloc()) alloc '(cptab->ctb_parts)': 64 at 0000000067593074. 00000400:00000010:0.0:1775764330.070462:0:2340:0:(lib-cpt.c:150:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 0000000062300c5c. 00000400:00000010:0.0:1775764330.070464:0:2340:0:(lib-cpt.c:154:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 8 at 0000000049b0e45c. 00000400:00000010:0.0:1775764330.070466:0:2340:0:(lib-cpt.c:150:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 00000000d6f45ddb. 00000400:00000010:0.0:1775764330.070468:0:2340:0:(lib-cpt.c:154:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 8 at 00000000f66b1490. 00000400:00000010:0.0:1775764330.846600:0:2340:0:(lock.c:50:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 0000000082f7e5fb. 00000400:00000010:0.0:1775764330.846611:0:2340:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005f92cf10. 00000400:00000010:0.0:1775764330.846616:0:2340:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000022d4de6e. 00000400:00000010:0.0:1775764330.846619:0:2340:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ceaadde2. 00000400:00000010:0.0:1775764330.846621:0:2340:0:(lock.c:50:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 0000000097c38783. 00000400:00000010:0.0:1775764330.846623:0:2340:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000dc7130b9. 00000400:00000010:0.0:1775764330.846625:0:2340:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000e3cd22b. 00000400:00000010:0.0:1775764330.846627:0:2340:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a00efab3. 00000400:00000200:0.0:1775764330.846728:0:2340:0:(api-ni.c:982:lnet_register_lnd()) lo LND registered 00000400:00000200:1.0F:1775764331.308836:0:2356:0:(api-ni.c:982:lnet_register_lnd()) tcp LND registered 00000020:02000000:2.0F:1775764332.049343:0:2369:0:(class_obd.c:784:obdclass_init()) Lustre: Build Version: 2.17.51_76_g43aed12 00000020:00000010:3.0F:1775764332.059452:0:2369:0:(lustre_handles.c:148:class_handle_init()) kmalloced '(handle_hash)': 2097152 at 00000000bd2de221. 00000001:00000010:3.0:1775764332.061211:0:2369:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000ce379979. 00000001:00000010:3.0:1775764332.061221:0:2369:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000004fc5084d. 00000001:00000010:3.0:1775764332.061225:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000fb0be0fa. 00000001:00000010:3.0:1775764332.061228:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000246e9e1b. 00000001:00000010:3.0:1775764332.061231:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000b0e62253. 00000001:00000010:3.0:1775764332.061233:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000cbe5ce2b. 00000020:00000010:3.0:1775764332.061283:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000afc47b45. 00000020:00000010:3.0:1775764332.061289:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000063ddbf6c. 00000020:00000010:3.0:1775764332.061292:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007b05ccc0. 00000020:00000010:3.0:1775764332.061294:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000009c453f9e. 00000001:00000010:3.0:1775764332.061296:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000f1d65e97. 00000020:00000010:3.0:1775764332.061386:0:2369:0:(cl_object.c:1176:cl_global_init()) kmalloced '(cl_envs)': 192 at 000000007fb8dfa8. 00000020:00000010:3.0:1775764332.061599:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b0aefbec. 00000020:00000010:3.0:1775764332.061602:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e7a90e31. 00000020:00000010:3.0:1775764332.061605:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000070e91593. 00000020:00000010:3.0:1775764332.061608:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006a10e4c1. 00000020:00000010:3.0:1775764332.061611:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000095fea115. 00000020:00000010:3.0:1775764332.061613:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000856b3db7. 00000020:00000010:3.0:1775764332.061615:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000ef835477. 00000020:00000010:3.0:1775764332.061618:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d766ed06. 00000020:00000010:3.0:1775764332.061620:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000065d49e7a. 00000020:00000010:3.0:1775764332.061621:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a631fe53. 00000020:00000010:3.0:1775764332.061623:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a9f5b8ec. 00000020:00000010:3.0:1775764332.061626:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007f31f217. 00000020:00000010:3.0:1775764332.061629:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000088cd7ed7. 00000020:00000010:3.0:1775764332.061631:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005388c6b8. 00000020:00000010:3.0:1775764332.061633:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000009e38efba. 00000020:00000010:3.0:1775764332.061635:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006b04944d. 02000000:00000010:3.0:1775764332.061640:0:2369:0:(page_pools.c:1086:obd_pool_init()) kmalloced '(page_pools)': 112 at 00000000f0ff0a1e. 02000000:00000010:3.0:1775764332.061644:0:2369:0:(page_pools.c:1090:obd_pool_init()) kmalloced '(pool_shrinkers)': 112 at 000000001d6a9f9c. 02000000:00000010:3.0:1775764332.061666:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 0000000064601a00. 02000000:00000010:3.0:1775764332.061671:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 1056 at 0000000075d654dc. 00000001:00000010:3.0:1775764332.061678:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000d2148a17. 00000020:00000010:3.0:1775764332.061699:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c9800ab4. 00000020:00000010:3.0:1775764332.061707:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c9800ab4. 02000000:00000010:3.0:1775764332.061711:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000696bf2c9. 02000000:00000010:3.0:1775764332.061714:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 528 at 000000004ada6057. 00000001:00000010:3.0:1775764332.061716:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000000b667e5a. 00000020:00000010:3.0:1775764332.061729:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c9800ab4. 00000020:00000010:3.0:1775764332.061736:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c9800ab4. 02000000:00000010:3.0:1775764332.061739:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000a196dac2. 02000000:00000010:3.0:1775764332.061741:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 264 at 00000000915ac28a. 00000001:00000010:3.0:1775764332.061743:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000012e85e56. 00000020:00000010:3.0:1775764332.061759:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c9800ab4. 00000020:00000010:3.0:1775764332.061766:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c9800ab4. 02000000:00000010:3.0:1775764332.061769:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000489ce02c. 02000000:00000010:3.0:1775764332.061771:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 136 at 00000000c9800ab4. 00000001:00000010:3.0:1775764332.061773:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000043165feb. 00000020:00000010:3.0:1775764332.061788:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062079:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062084:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000b20743c2. 02000000:00000010:3.0:1775764332.062087:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 72 at 000000001992beee. 00000001:00000010:3.0:1775764332.062089:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000006a98f02. 00000020:00000010:3.0:1775764332.062110:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062118:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062122:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000008a58501a. 02000000:00000010:3.0:1775764332.062124:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 40 at 00000000c5b2b37c. 00000001:00000010:3.0:1775764332.062126:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000a6f8ac03. 00000020:00000010:3.0:1775764332.062142:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062148:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062151:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000626afe28. 02000000:00000010:3.0:1775764332.062154:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 24 at 00000000cfc37b0c. 00000001:00000010:3.0:1775764332.062155:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000b7494b66. 00000020:00000010:3.0:1775764332.062178:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062184:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062187:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000001ac39025. 02000000:00000010:3.0:1775764332.062189:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 16 at 000000001ad43cf5. 00000001:00000010:3.0:1775764332.062190:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000d8d0ff45. 00000020:00000010:3.0:1775764332.062207:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062215:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062218:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000006098e330. 02000000:00000010:3.0:1775764332.062221:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 000000001563f748. 00000001:00000010:3.0:1775764332.062223:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000c1ffce0a. 00000020:00000010:3.0:1775764332.062242:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062276:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062281:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000d3394329. 02000000:00000010:3.0:1775764332.062284:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 000000000c89be7d. 00000001:00000010:3.0:1775764332.062286:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000005ebf6f45. 00000020:00000010:3.0:1775764332.062307:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062313:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062316:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000005b2a2d72. 02000000:00000010:3.0:1775764332.062318:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 000000009e6a102c. 00000001:00000010:3.0:1775764332.062319:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000b8f64ca2. 00000020:00000010:3.0:1775764332.062333:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062339:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062341:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000004597f28e. 02000000:00000010:3.0:1775764332.062342:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 00000000141844b3. 00000001:00000010:3.0:1775764332.062345:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000ffa713b2. 00000020:00000010:3.0:1775764332.062359:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062364:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062367:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000e427ec5a. 02000000:00000010:3.0:1775764332.062368:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 0000000027024005. 00000001:00000010:3.0:1775764332.062371:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000798cb44f. 00000020:00000010:3.0:1775764332.062403:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062414:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 02000000:00000010:3.0:1775764332.062417:0:2369:0:(page_pools.c:1097:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000c6c31f9e. 02000000:00000010:3.0:1775764332.062420:0:2369:0:(page_pools.c:1057:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 000000001ebe3683. 00000001:00000010:3.0:1775764332.062422:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000153f6202. 00000020:00000010:3.0:1775764332.062439:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000027409528. 00000020:00000010:3.0:1775764332.062479:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000027409528. 00000400:00000010:2.0:1775764333.418427:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000055d9e39c. 00000400:00000010:2.0:1775764333.418439:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003c9c974a. 00000400:00000010:2.0:1775764333.418442:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000072d83629. 00000100:00000010:2.0:1775764333.418448:0:2384:0:(service.c:3571:ptlrpc_hr_init()) kmalloced '(hrp->hrp_thrs)': 96 at 000000006e9c0610. 00000100:00000010:2.0:1775764333.418451:0:2384:0:(service.c:3571:ptlrpc_hr_init()) kmalloced '(hrp->hrp_thrs)': 96 at 000000006b5c7825. 00000100:00000010:0.0:1775764333.419128:0:2386:0:(service.c:3214:ptlrpc_hr_main()) kmalloced '(env)': 56 at 000000001fe0f80f. 00000020:00000010:0.0:1775764333.419146:0:2386:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b07b312. 00000020:00000010:0.0:1775764333.419151:0:2386:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b04859a7. 00000040:00000010:0.0:1775764333.419155:0:2386:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000007da8296a. 00000020:00000010:0.0:1775764333.419159:0:2386:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000b6b05426. 00000020:00000010:0.0:1775764333.419163:0:2386:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000e7857562. 00000100:00000010:0.0:1775764333.419627:0:2387:0:(service.c:3214:ptlrpc_hr_main()) kmalloced '(env)': 56 at 000000005e91d7f5. 00000020:00000010:2.0:1775764333.419991:0:2387:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001cd903cd. 00000020:00000010:2.0:1775764333.419996:0:2387:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000002a8b3f78. 00000040:00000010:2.0:1775764333.420000:0:2387:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000054ce8667. 00000020:00000010:2.0:1775764333.420004:0:2387:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000086cf995a. 00000020:00000010:2.0:1775764333.420009:0:2387:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000c73ccc62. 00000100:00000200:0.0:1775764333.420120:0:2384:0:(events.c:596:ptlrpc_ni_init()) My pid is: 3039 00000400:00000010:0.0:1775764333.420353:0:2384:0:(api-ni.c:679:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 00000000ae301e15. 00000400:00000010:0.0:1775764333.420358:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000e3ab3cd9. 00000400:00000010:0.0:1775764333.420362:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000d235bd88. 00000400:00000010:0.0:1775764333.420365:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000062a202ab. 00000400:00000010:0.0:1775764333.420368:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001e3c1c36. 00000400:00000010:0.0:1775764333.420370:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000045880b0c. 00000400:00000010:0.0:1775764333.420372:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000db08d0c9. 00000400:00000010:0.0:1775764333.420377:0:2384:0:(peer.c:108:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000006608aa06. 00000400:00000010:0.0:1775764333.420387:0:2384:0:(peer.c:108:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000005c4d3996. 00000400:00000010:0.0:1775764333.420391:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000042f09b5c. 00000400:00000010:0.0:1775764333.420393:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000fed9ae78. 00000400:00000010:0.0:1775764333.420395:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000668a372e. 00000400:00000010:0.0:1775764333.420399:0:2384:0:(lib-msg.c:1260:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 16 at 0000000055257c06. 00000400:00000010:0.0:1775764333.420402:0:2384:0:(lib-msg.c:1270:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 16 at 000000009461b535. 00000400:00000010:0.0:1775764333.420404:0:2384:0:(lib-msg.c:1260:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 16 at 0000000078818554. 00000400:00000010:0.0:1775764333.420405:0:2384:0:(lib-msg.c:1270:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 16 at 000000004e75dff1. 00000400:00000010:0.0:1775764333.420647:0:2384:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000e09a86c3. 00000400:00000010:0.0:1775764333.420678:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000991429af. 00000400:00000010:0.0:1775764333.420681:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000009747db7. 00000400:00000010:0.0:1775764333.420683:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d23c597b. 00000400:00000010:0.0:1775764333.420966:0:2384:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 000000006fb9b9f8. 00000400:00000010:0.0:1775764333.421042:0:2384:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000cc9d8154. 00000400:00000010:0.0:1775764333.421063:0:2384:0:(lib-ptl.c:867:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 000000005372ff6a. 00000400:00000010:0.0:1775764333.421067:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e212227c. 00000400:00000010:0.0:1775764333.421069:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f15ab6bf. 00000400:00000010:0.0:1775764333.421072:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000046868285. 00000400:00000010:0.0:1775764333.421073:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000d587cd0. 00000400:00000010:0.0:1775764333.421079:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000036babb55. 00000400:00000010:0.0:1775764333.421084:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003cff8685. 00000400:00000010:0.0:1775764333.421087:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000004c53698f. 00000400:00000010:0.0:1775764333.421089:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000ece38f24. 00000400:00000010:0.0:1775764333.421091:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000056bd458d. 00000400:00000010:0.0:1775764333.421093:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ec06fc99. 00000400:00000010:0.0:1775764333.421106:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000028c3f70a. 00000400:00000010:0.0:1775764333.421111:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009ee3e9fa. 00000400:00000010:0.0:1775764333.421114:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000093bf9118. 00000400:00000010:0.0:1775764333.421117:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000ba473a99. 00000400:00000010:0.0:1775764333.421119:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007e576504. 00000400:00000010:0.0:1775764333.421121:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bcaf30f7. 00000400:00000010:0.0:1775764333.421128:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a561e05c. 00000400:00000010:0.0:1775764333.421132:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005f98badf. 00000400:00000010:0.0:1775764333.421135:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000082b6be05. 00000400:00000010:0.0:1775764333.421136:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f48a0b00. 00000400:00000010:0.0:1775764333.421137:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000062ea5392. 00000400:00000010:0.0:1775764333.421139:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008fcf19ae. 00000400:00000010:0.0:1775764333.421148:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000edd6784d. 00000400:00000010:0.0:1775764333.421154:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007bcfcf13. 00000400:00000010:0.0:1775764333.421157:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000011d217a5. 00000400:00000010:0.0:1775764333.421159:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000083e367d7. 00000400:00000010:0.0:1775764333.421161:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b08ff930. 00000400:00000010:0.0:1775764333.421163:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c4230d65. 00000400:00000010:0.0:1775764333.421167:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000079b7ecf8. 00000400:00000010:0.0:1775764333.421173:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007978bbc7. 00000400:00000010:0.0:1775764333.421175:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c6de5280. 00000400:00000010:0.0:1775764333.421177:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005bd0c2ce. 00000400:00000010:0.0:1775764333.421178:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a9044e4f. 00000400:00000010:0.0:1775764333.421180:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006606438d. 00000400:00000010:0.0:1775764333.421190:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004432718a. 00000400:00000010:0.0:1775764333.421195:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002717727c. 00000400:00000010:0.0:1775764333.421198:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000248e7871. 00000400:00000010:0.0:1775764333.421199:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f5e3e743. 00000400:00000010:0.0:1775764333.421201:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fd9a09b3. 00000400:00000010:0.0:1775764333.421203:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b81e5078. 00000400:00000010:0.0:1775764333.421206:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000012401a14. 00000400:00000010:0.0:1775764333.421211:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000076d3c8ce. 00000400:00000010:0.0:1775764333.421213:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b475f98f. 00000400:00000010:0.0:1775764333.421215:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000017143619. 00000400:00000010:0.0:1775764333.421217:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007e27b989. 00000400:00000010:0.0:1775764333.421218:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000680f4ff7. 00000400:00000010:0.0:1775764333.421228:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000012f0be19. 00000400:00000010:0.0:1775764333.421234:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009a23988a. 00000400:00000010:0.0:1775764333.421237:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000005e8400c5. 00000400:00000010:0.0:1775764333.421244:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000057ec8815. 00000400:00000010:0.0:1775764333.421246:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d0168c32. 00000400:00000010:0.0:1775764333.421248:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001e1ac901. 00000400:00000010:0.0:1775764333.421251:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c1cdb771. 00000400:00000010:0.0:1775764333.421256:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c4f2521a. 00000400:00000010:0.0:1775764333.421259:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e4c9fd4c. 00000400:00000010:0.0:1775764333.421261:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005cb00efb. 00000400:00000010:0.0:1775764333.421263:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d3e7c53f. 00000400:00000010:0.0:1775764333.421264:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000069c56525. 00000400:00000010:0.0:1775764333.421274:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000125f400b. 00000400:00000010:0.0:1775764333.421279:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a160e617. 00000400:00000010:0.0:1775764333.421282:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008c388481. 00000400:00000010:0.0:1775764333.421284:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d1b7e38e. 00000400:00000010:0.0:1775764333.421286:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e5f1083b. 00000400:00000010:0.0:1775764333.421288:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000db469f93. 00000400:00000010:0.0:1775764333.421292:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008aba18f6. 00000400:00000010:0.0:1775764333.421296:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f8f5a88d. 00000400:00000010:0.0:1775764333.421299:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000006897d659. 00000400:00000010:0.0:1775764333.421301:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000e47b8909. 00000400:00000010:0.0:1775764333.421304:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000099b7852b. 00000400:00000010:0.0:1775764333.421306:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000070901d3b. 00000400:00000010:0.0:1775764333.421317:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000915e6d1b. 00000400:00000010:0.0:1775764333.421323:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007695f171. 00000400:00000010:0.0:1775764333.421326:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e76cf231. 00000400:00000010:0.0:1775764333.421328:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d673711c. 00000400:00000010:0.0:1775764333.421330:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006027e67b. 00000400:00000010:0.0:1775764333.421332:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d7911fa5. 00000400:00000010:0.0:1775764333.421337:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c1304f99. 00000400:00000010:0.0:1775764333.421343:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000089404ec8. 00000400:00000010:0.0:1775764333.421345:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007f09ac63. 00000400:00000010:0.0:1775764333.421346:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000054be31d8. 00000400:00000010:0.0:1775764333.421348:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d6d42fe5. 00000400:00000010:0.0:1775764333.421350:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001631657b. 00000400:00000010:0.0:1775764333.421362:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000073946a15. 00000400:00000010:0.0:1775764333.421368:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fb5072cc. 00000400:00000010:0.0:1775764333.421370:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b21c4c00. 00000400:00000010:0.0:1775764333.421372:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000936d7f01. 00000400:00000010:0.0:1775764333.421373:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a3dc0fa5. 00000400:00000010:0.0:1775764333.421375:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000021c269d0. 00000400:00000010:0.0:1775764333.421379:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000719cc07a. 00000400:00000010:0.0:1775764333.421384:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009bb4355c. 00000400:00000010:0.0:1775764333.421387:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000541319f1. 00000400:00000010:0.0:1775764333.421388:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000007abaeec5. 00000400:00000010:0.0:1775764333.421390:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000841c8bc8. 00000400:00000010:0.0:1775764333.421392:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fae90108. 00000400:00000010:0.0:1775764333.421402:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003af6fc89. 00000400:00000010:0.0:1775764333.421409:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004930174c. 00000400:00000010:0.0:1775764333.421412:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b9611425. 00000400:00000010:0.0:1775764333.421414:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000040133ccb. 00000400:00000010:0.0:1775764333.421416:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000085dec414. 00000400:00000010:0.0:1775764333.421418:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000017abd18c. 00000400:00000010:0.0:1775764333.421422:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000032cfffdc. 00000400:00000010:0.0:1775764333.421428:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000afc937b3. 00000400:00000010:0.0:1775764333.421430:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000d3f08813. 00000400:00000010:0.0:1775764333.421432:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000078bebb0a. 00000400:00000010:0.0:1775764333.421434:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fc1c651a. 00000400:00000010:0.0:1775764333.421436:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002dc234fa. 00000400:00000010:0.0:1775764333.421446:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ebecb1c7. 00000400:00000010:0.0:1775764333.421451:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000025f3238e. 00000400:00000010:0.0:1775764333.421454:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000d095e9cb. 00000400:00000010:0.0:1775764333.421456:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000008ca4f1cb. 00000400:00000010:0.0:1775764333.421578:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a67f91af. 00000400:00000010:0.0:1775764333.421580:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000997a72d2. 00000400:00000010:0.0:1775764333.421584:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b82d46de. 00000400:00000010:0.0:1775764333.421588:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000068be730c. 00000400:00000010:0.0:1775764333.421590:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003fafdffd. 00000400:00000010:0.0:1775764333.421597:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000000021a5b5. 00000400:00000010:0.0:1775764333.421600:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000036fa419a. 00000400:00000010:0.0:1775764333.421601:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000017085cb3. 00000400:00000010:0.0:1775764333.421612:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000aa69a06e. 00000400:00000010:0.0:1775764333.421618:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002bfae621. 00000400:00000010:0.0:1775764333.421621:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000009492670. 00000400:00000010:0.0:1775764333.421622:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000e981ce7e. 00000400:00000010:0.0:1775764333.421624:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007af89e05. 00000400:00000010:0.0:1775764333.421626:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000044721146. 00000400:00000010:0.0:1775764333.421631:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b7e207df. 00000400:00000010:0.0:1775764333.421636:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000778a8a9. 00000400:00000010:0.0:1775764333.421639:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000090fc3fd3. 00000400:00000010:0.0:1775764333.421641:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000dba0a102. 00000400:00000010:0.0:1775764333.421643:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a748c5ff. 00000400:00000010:0.0:1775764333.421645:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000013f9a3c2. 00000400:00000010:0.0:1775764333.421654:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f6c66763. 00000400:00000010:0.0:1775764333.421659:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000738363a. 00000400:00000010:0.0:1775764333.421671:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003a795966. 00000400:00000010:0.0:1775764333.421673:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c0e74e77. 00000400:00000010:0.0:1775764333.421675:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b34e058c. 00000400:00000010:0.0:1775764333.421677:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f8aceefd. 00000400:00000010:0.0:1775764333.421681:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e1dccd25. 00000400:00000010:0.0:1775764333.421686:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006af56132. 00000400:00000010:0.0:1775764333.421688:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000965d7930. 00000400:00000010:0.0:1775764333.421690:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000039078dda. 00000400:00000010:0.0:1775764333.421691:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004c09807e. 00000400:00000010:0.0:1775764333.421693:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a44be70a. 00000400:00000010:0.0:1775764333.421703:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b318414b. 00000400:00000010:0.0:1775764333.421709:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f00c0bec. 00000400:00000010:0.0:1775764333.421712:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000439c396b. 00000400:00000010:0.0:1775764333.421713:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000021942a67. 00000400:00000010:0.0:1775764333.421715:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008bdb8197. 00000400:00000010:0.0:1775764333.421717:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000092115a14. 00000400:00000010:0.0:1775764333.421725:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bf8b0a97. 00000400:00000010:0.0:1775764333.421729:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009eae2b0d. 00000400:00000010:0.0:1775764333.421732:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000480e1f4e. 00000400:00000010:0.0:1775764333.421959:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005fba2697. 00000400:00000010:0.0:1775764333.421963:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000029606e24. 00000400:00000010:0.0:1775764333.421965:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000638583d9. 00000400:00000010:0.0:1775764333.421976:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f153b60e. 00000400:00000010:0.0:1775764333.421981:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fe8799cb. 00000400:00000010:0.0:1775764333.421984:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007ab45966. 00000400:00000010:0.0:1775764333.421986:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000cc60b885. 00000400:00000010:0.0:1775764333.421988:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000933b6eda. 00000400:00000010:0.0:1775764333.421990:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dc2bbfd6. 00000400:00000010:0.0:1775764333.421994:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009dd63462. 00000400:00000010:0.0:1775764333.421998:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ca3e554a. 00000400:00000010:0.0:1775764333.422000:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000ff2cb247. 00000400:00000010:0.0:1775764333.422002:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000636fa214. 00000400:00000010:0.0:1775764333.422004:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004224d84d. 00000400:00000010:0.0:1775764333.422005:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004f32fa00. 00000400:00000010:0.0:1775764333.422015:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007a2fbf6a. 00000400:00000010:0.0:1775764333.422021:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000074a95b02. 00000400:00000010:0.0:1775764333.422024:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b1cad5be. 00000400:00000010:0.0:1775764333.422026:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000adcd2f15. 00000400:00000010:0.0:1775764333.422027:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fa6d085e. 00000400:00000010:0.0:1775764333.422029:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001e8df836. 00000400:00000010:0.0:1775764333.422032:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000071ef4315. 00000400:00000010:0.0:1775764333.422036:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f725802c. 00000400:00000010:0.0:1775764333.422039:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008b80a4d5. 00000400:00000010:0.0:1775764333.422040:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000082c8ade4. 00000400:00000010:0.0:1775764333.422042:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001a7677c0. 00000400:00000010:0.0:1775764333.422043:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000048b0a660. 00000400:00000010:0.0:1775764333.422055:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cd0451e3. 00000400:00000010:0.0:1775764333.422061:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ffb851d2. 00000400:00000010:0.0:1775764333.422064:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000037771218. 00000400:00000010:0.0:1775764333.422070:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004773ccac. 00000400:00000010:0.0:1775764333.422071:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000089314b57. 00000400:00000010:0.0:1775764333.422073:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005be8e940. 00000400:00000010:0.0:1775764333.422078:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008f4959f2. 00000400:00000010:0.0:1775764333.422082:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a021a10b. 00000400:00000010:0.0:1775764333.422085:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000004dd3a9de. 00000400:00000010:0.0:1775764333.422086:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000776e2f84. 00000400:00000010:0.0:1775764333.422088:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007d080326. 00000400:00000010:0.0:1775764333.422089:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000098658d90. 00000400:00000010:0.0:1775764333.422098:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001b6fe45f. 00000400:00000010:0.0:1775764333.422103:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000085c9b444. 00000400:00000010:0.0:1775764333.422105:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003c47662f. 00000400:00000010:0.0:1775764333.422108:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d670aa36. 00000400:00000010:0.0:1775764333.422110:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006f8e7643. 00000400:00000010:0.0:1775764333.422112:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007121c0a5. 00000400:00000010:0.0:1775764333.422115:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e38e8ec1. 00000400:00000010:0.0:1775764333.422119:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000856ee862. 00000400:00000010:0.0:1775764333.422123:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000144835d4. 00000400:00000010:0.0:1775764333.422124:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d9445895. 00000400:00000010:0.0:1775764333.422126:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fff1a88e. 00000400:00000010:0.0:1775764333.422128:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003fd5c80c. 00000400:00000010:0.0:1775764333.422139:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e0be5449. 00000400:00000010:0.0:1775764333.422146:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ac9ca005. 00000400:00000010:0.0:1775764333.422148:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008d513f4e. 00000400:00000010:0.0:1775764333.422150:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000ddef40dd. 00000400:00000010:0.0:1775764333.422152:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008f5885ab. 00000400:00000010:0.0:1775764333.422154:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000f1d749f. 00000400:00000010:0.0:1775764333.422159:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006098e966. 00000400:00000010:0.0:1775764333.422165:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003f031757. 00000400:00000010:0.0:1775764333.422167:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000ac395da6. 00000400:00000010:0.0:1775764333.422169:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d71893d4. 00000400:00000010:0.0:1775764333.422171:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000086946137. 00000400:00000010:0.0:1775764333.422173:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000007454524. 00000400:00000010:0.0:1775764333.422185:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000259bf3e5. 00000400:00000010:0.0:1775764333.422190:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005f7f3177. 00000400:00000010:0.0:1775764333.422193:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000a4936904. 00000400:00000010:0.0:1775764333.422195:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005a590c2e. 00000400:00000010:0.0:1775764333.422196:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006d987554. 00000400:00000010:0.0:1775764333.422198:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000811661b2. 00000400:00000010:0.0:1775764333.422202:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000295174e. 00000400:00000010:0.0:1775764333.422206:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000861e14dd. 00000400:00000010:0.0:1775764333.422208:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000db424770. 00000400:00000010:0.0:1775764333.422210:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000003dcfbc6c. 00000400:00000010:0.0:1775764333.422211:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f00c87b3. 00000400:00000010:0.0:1775764333.422213:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000766e54eb. 00000400:00000010:0.0:1775764333.422222:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009ebdfc61. 00000400:00000010:0.0:1775764333.422227:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eb035739. 00000400:00000010:0.0:1775764333.422230:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f2209859. 00000400:00000010:0.0:1775764333.422231:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000092c644a7. 00000400:00000010:0.0:1775764333.422233:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000179cdde3. 00000400:00000010:0.0:1775764333.422235:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e3b8bfe7. 00000400:00000010:0.0:1775764333.422239:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c635a5bf. 00000400:00000010:0.0:1775764333.422243:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004f95723e. 00000400:00000010:0.0:1775764333.422246:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000009d233d11. 00000400:00000010:0.0:1775764333.422248:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000003d5e3666. 00000400:00000010:0.0:1775764333.422250:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fbad8cd2. 00000400:00000010:0.0:1775764333.422251:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a28efb55. 00000400:00000010:0.0:1775764333.422261:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000def31baf. 00000400:00000010:0.0:1775764333.422266:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002756d628. 00000400:00000010:0.0:1775764333.422270:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000ab992c43. 00000400:00000010:0.0:1775764333.422272:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000025ca86c8. 00000400:00000010:0.0:1775764333.422274:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f8ac4ff1. 00000400:00000010:0.0:1775764333.422275:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c5b4ff7b. 00000400:00000010:0.0:1775764333.422279:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000086cbf661. 00000400:00000010:0.0:1775764333.422283:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c8ee2c61. 00000400:00000010:0.0:1775764333.422286:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000030d15dad. 00000400:00000010:0.0:1775764333.422291:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a5254116. 00000400:00000010:0.0:1775764333.422293:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000056394b34. 00000400:00000010:0.0:1775764333.422295:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000007f1dc6d. 00000400:00000010:0.0:1775764333.422305:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c2e9e1dc. 00000400:00000010:0.0:1775764333.422309:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003c4f1127. 00000400:00000010:0.0:1775764333.422312:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000045d4bb40. 00000400:00000010:0.0:1775764333.422314:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a4c4f881. 00000400:00000010:0.0:1775764333.422316:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000544658a6. 00000400:00000010:0.0:1775764333.422318:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005c93d745. 00000400:00000010:0.0:1775764333.422322:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004424370b. 00000400:00000010:0.0:1775764333.422326:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dccebd3e. 00000400:00000010:0.0:1775764333.422328:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000072707c1b. 00000400:00000010:0.0:1775764333.422330:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000fecf48be. 00000400:00000010:0.0:1775764333.422332:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000033ba26c6. 00000400:00000010:0.0:1775764333.422334:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000de7579cc. 00000400:00000010:0.0:1775764333.422343:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eb7fbc88. 00000400:00000010:0.0:1775764333.422348:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f713f0a2. 00000400:00000010:0.0:1775764333.422365:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c0041ef2. 00000400:00000010:0.0:1775764333.422367:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f3c9eb9a. 00000400:00000010:0.0:1775764333.422369:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000066a8903f. 00000400:00000010:0.0:1775764333.422371:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000086a45b8. 00000400:00000010:0.0:1775764333.422376:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000069b1d433. 00000400:00000010:0.0:1775764333.422380:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007f369497. 00000400:00000010:0.0:1775764333.422383:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000039206a78. 00000400:00000010:0.0:1775764333.422385:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000908d3e66. 00000400:00000010:0.0:1775764333.422387:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bb1caac4. 00000400:00000010:0.0:1775764333.422388:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ae5812bc. 00000400:00000010:0.0:1775764333.422397:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000027f5b4d3. 00000400:00000010:0.0:1775764333.422401:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e5179586. 00000400:00000010:0.0:1775764333.422404:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000cc9f8ff4. 00000400:00000010:0.0:1775764333.422405:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004f944126. 00000400:00000010:0.0:1775764333.422407:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a93a4a77. 00000400:00000010:0.0:1775764333.422409:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000094d817ab. 00000400:00000010:0.0:1775764333.422415:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bd7e4c59. 00000400:00000010:0.0:1775764333.422420:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004330f44b. 00000400:00000010:0.0:1775764333.422422:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c276b82d. 00000400:00000010:0.0:1775764333.422424:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a44e3496. 00000400:00000010:0.0:1775764333.422426:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006b292ea6. 00000400:00000010:0.0:1775764333.422427:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000969bdc26. 00000400:00000010:0.0:1775764333.422437:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000072881775. 00000400:00000010:0.0:1775764333.422443:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d2043e58. 00000400:00000010:0.0:1775764333.422446:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b875b0a8. 00000400:00000010:0.0:1775764333.422448:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000dabe1543. 00000400:00000010:0.0:1775764333.422450:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a120c141. 00000400:00000010:0.0:1775764333.422452:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006eb59d89. 00000400:00000010:0.0:1775764333.422456:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000019951485. 00000400:00000010:0.0:1775764333.423257:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c8835be7. 00000400:00000010:0.0:1775764333.423264:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000009011ba04. 00000400:00000010:0.0:1775764333.423266:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001958d4f4. 00000400:00000010:0.0:1775764333.423269:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dffa8a1d. 00000400:00000010:0.0:1775764333.423271:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000061af3e95. 00000400:00000010:0.0:1775764333.423288:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000af7cea4. 00000400:00000010:0.0:1775764333.423295:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000070beebcc. 00000400:00000010:0.0:1775764333.423298:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000012789bf6. 00000400:00000010:0.0:1775764333.423299:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000250915cf. 00000400:00000010:0.0:1775764333.423301:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000063260b96. 00000400:00000010:0.0:1775764333.423303:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001fb8c8ba. 00000400:00000010:0.0:1775764333.423307:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000058ce0b74. 00000400:00000010:0.0:1775764333.423311:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000099b21a7b. 00000400:00000010:0.0:1775764333.423313:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000591af337. 00000400:00000010:0.0:1775764333.423314:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001308bc32. 00000400:00000010:0.0:1775764333.423316:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cfa35761. 00000400:00000010:0.0:1775764333.423317:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000345e1856. 00000400:00000010:0.0:1775764333.423325:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000089d1587a. 00000400:00000010:0.0:1775764333.423330:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b732f731. 00000400:00000010:0.0:1775764333.423332:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000efffd749. 00000400:00000010:0.0:1775764333.423340:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000081db6b0e. 00000400:00000010:0.0:1775764333.423341:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006629f086. 00000400:00000010:0.0:1775764333.423343:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009ace23a9. 00000400:00000010:0.0:1775764333.423347:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000120469d5. 00000400:00000010:0.0:1775764333.423351:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006f2f8b01. 00000400:00000010:0.0:1775764333.423353:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007dad10c8. 00000400:00000010:0.0:1775764333.423354:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000bfeb0785. 00000400:00000010:0.0:1775764333.423357:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000642eb439. 00000400:00000010:0.0:1775764333.423358:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ee35695a. 00000400:00000010:0.0:1775764333.423367:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000909592b7. 00000400:00000010:0.0:1775764333.423372:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000da85c0cd. 00000400:00000010:0.0:1775764333.423374:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000001cb1f09b. 00000400:00000010:0.0:1775764333.423376:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000087fece36. 00000400:00000010:0.0:1775764333.423378:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005279eb6d. 00000400:00000010:0.0:1775764333.423380:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003acb46f1. 00000400:00000010:0.0:1775764333.423384:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004f3d49ce. 00000400:00000010:0.0:1775764333.423388:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000511e9a2a. 00000400:00000010:0.0:1775764333.423391:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000001fd0271f. 00000400:00000010:0.0:1775764333.423393:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000754c8396. 00000400:00000010:0.0:1775764333.423394:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000025b81887. 00000400:00000010:0.0:1775764333.423396:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000009b453e. 00000400:00000010:0.0:1775764333.423405:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000050b4b67b. 00000400:00000010:0.0:1775764333.423411:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000039c6bf. 00000400:00000010:0.0:1775764333.423414:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000ad1a7eed. 00000400:00000010:0.0:1775764333.423416:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a9701e9c. 00000400:00000010:0.0:1775764333.423417:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b4fbbae6. 00000400:00000010:0.0:1775764333.423419:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000069cbe9b9. 00000400:00000010:0.0:1775764333.423423:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000246cc25. 00000400:00000010:0.0:1775764333.423427:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d5254ab8. 00000400:00000010:0.0:1775764333.423430:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f804a576. 00000400:00000010:0.0:1775764333.423432:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000007e639004. 00000400:00000010:0.0:1775764333.423433:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c1a82450. 00000400:00000010:0.0:1775764333.423435:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009dd9777f. 00000400:00000010:0.0:1775764333.423448:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c1d26a14. 00000400:00000010:0.0:1775764333.423454:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000208f70a1. 00000400:00000010:0.0:1775764333.423609:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000fc8e2517. 00000400:00000010:0.0:1775764333.423614:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000601df4c1. 00000400:00000010:0.0:1775764333.423616:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007cb49d7d. 00000400:00000010:0.0:1775764333.423618:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000031b897d8. 00000400:00000010:0.0:1775764333.423622:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008ff0579c. 00000400:00000010:0.0:1775764333.423627:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005c7de9dc. 00000400:00000010:0.0:1775764333.423629:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008c904c40. 00000400:00000010:0.0:1775764333.423631:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000008747bef4. 00000400:00000010:0.0:1775764333.423633:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e2369026. 00000400:00000010:0.0:1775764333.423635:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a6dd1ef2. 00000400:00000010:0.0:1775764333.423646:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008686d1a0. 00000400:00000010:0.0:1775764333.423652:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000093948107. 00000400:00000010:0.0:1775764333.423655:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f5a549e2. 00000400:00000010:0.0:1775764333.423893:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a2cb0792. 00000400:00000010:0.0:1775764333.423898:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008706aaa7. 00000400:00000010:0.0:1775764333.423900:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007234e5c3. 00000400:00000010:0.0:1775764333.423904:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ab9b12f1. 00000400:00000010:0.0:1775764333.423909:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cf549a15. 00000400:00000010:0.0:1775764333.423912:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e9e31c40. 00000400:00000010:0.0:1775764333.423914:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000527d87fb. 00000400:00000010:0.0:1775764333.423916:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000002ed25ca. 00000400:00000010:0.0:1775764333.423919:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000045918bb1. 00000400:00000010:0.0:1775764333.423931:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007c62d4bb. 00000400:00000010:0.0:1775764333.423937:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000070a0d17c. 00000400:00000010:0.0:1775764333.423940:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000dc67709d. 00000400:00000010:0.0:1775764333.423941:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f1ca4d0e. 00000400:00000010:0.0:1775764333.423943:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007b7f9ff8. 00000400:00000010:0.0:1775764333.423945:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c5355de3. 00000400:00000010:0.0:1775764333.423949:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f68476c1. 00000400:00000010:0.0:1775764333.423954:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000054329dac. 00000400:00000010:0.0:1775764333.423957:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003ffc4d28. 00000400:00000010:0.0:1775764333.423963:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d446a850. 00000400:00000010:0.0:1775764333.423966:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007288e36d. 00000400:00000010:0.0:1775764333.423968:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b152dbd5. 00000400:00000010:0.0:1775764333.423978:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eac491e6. 00000400:00000010:0.0:1775764333.423985:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004d36b0cc. 00000400:00000010:0.0:1775764333.423988:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000012029375. 00000400:00000010:0.0:1775764333.423991:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ffd43d48. 00000400:00000010:0.0:1775764333.423992:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c39f1c20. 00000400:00000010:0.0:1775764333.423997:0:2384:0:(config.c:344:lnet_net_alloc()) alloc '(net)': 176 at 0000000006a3268a. 00000400:00000010:0.0:1775764333.424002:0:2384:0:(config.c:425:lnet_ni_alloc_common()) alloc '(ni)': 448 at 000000007d1eafb7. 00000400:00000010:0.0:1775764333.424005:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000515d12d4. 00000400:00000010:0.0:1775764333.424007:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002a9887e7. 00000400:00000010:0.0:1775764333.424008:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ff9f1bc0. 00000400:00000010:0.0:1775764333.424010:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c0b8bec9. 00000400:00000010:0.0:1775764333.424012:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f48953c6. 00000400:00000010:0.0:1775764333.424014:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003a55a049. 00000400:00000010:0.0:1775764333.424021:0:2384:0:(config.c:592:lnet_parse_networks()) alloc '(tokens)': 4 at 00000000d45611fe. 00000400:00000010:0.0:1775764333.424026:0:2384:0:(config.c:344:lnet_net_alloc()) alloc '(net)': 176 at 0000000037da631b. 00000400:00000010:0.0:1775764333.424029:0:2384:0:(config.c:425:lnet_ni_alloc_common()) alloc '(ni)': 448 at 000000002f0d5031. 00000400:00000010:0.0:1775764333.424031:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000081dd1b5c. 00000400:00000010:0.0:1775764333.424033:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fc6cd0f0. 00000400:00000010:0.0:1775764333.424035:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001a431e61. 00000400:00000010:0.0:1775764333.425425:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000007b01027c. 00000400:00000010:0.0:1775764333.425442:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a6fa4a4c. 00000400:00000010:0.0:1775764333.425445:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007206da25. 00000400:00000010:0.0:1775764333.425449:0:2384:0:(config.c:804:lnet_parse_networks()) kfreed 'tokens': 4 at 00000000d45611fe. 00000400:00000010:0.0:1775764333.425555:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b602adfc. 00000400:00000010:0.0:1775764333.425559:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000bd2c2784. 00000400:00000010:0.0:1775764333.425561:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000058ff67c. 00000800:00000200:3.0:1775764333.432133:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 000000005b751044 00000800:00000200:3.0:1775764333.432143:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 000000005b751044 00000800:00000200:3.0:1775764333.432146:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 000000005b751044 00000800:00000200:3.0:1775764333.432148:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 000000005b751044 00000800:00000010:3.0:1775764333.432156:0:2384:0:(socklnd.c:2622:ksocknal_startup()) alloc '(net)': 208 at 00000000d38d11fa. 00000400:00000010:3.0:1775764333.432168:0:2384:0:(config.c:398:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 00000000f0826160. 00000800:00000200:3.0:1775764333.432174:0:2384:0:(socklnd_modparams.c:304:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000800:00000010:2.0:1775764333.432929:0:2393:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 00000000cfedff8b. 00000800:00000010:2.0:1775764333.432936:0:2393:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 0000000009f72c69. 00000800:00000010:3.0:1775764333.433098:0:2394:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 0000000062bbbee5. 00000800:00000010:3.0:1775764333.433104:0:2394:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 0000000084261a4c. 00000800:00000010:3.0:1775764333.433392:0:2395:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 00000000ac703f59. 00000800:00000010:3.0:1775764333.433397:0:2395:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 000000000316e13d. 00000400:00000200:2.0:1775764333.433476:0:2384:0:(config.c:1527:lnet_get_link_status_locked()) device idx 1 get_link 2 00000400:00000200:2.0:1775764333.433589:0:2384:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.202.4) 00000400:02000000:2.0:1775764333.433607:0:2384:0:(api-ni.c:2704:lnet_startup_lndni()) Added LNI 192.168.202.4@tcp [8/256/0/180] 00000800:00000010:3.0:1775764333.433974:0:2396:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 00000000f16e0da1. 00000800:00000010:3.0:1775764333.433980:0:2396:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 000000004c60f0d1. 00000400:00000010:3.0:1775764333.445641:0:2384:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000763bbc69. 00000400:00000010:3.0:1775764333.445653:0:2384:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ac83ef5a. 00000400:00000010:3.0:1775764333.445685:0:2384:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000877cca33. 00000400:00000200:3.0:1775764333.445698:0:2384:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 0 lazy 00000400:00000010:3.0:1775764333.445702:0:2384:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000128152f3. 00000400:00000010:3.0:1775764333.445705:0:2384:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009e70d837. 00000400:00000010:3.0:1775764333.445707:0:2384:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c30a1203. 00000400:00000200:3.0:1775764333.445709:0:2384:0:(api-ni.c:2331:lnet_push_target_post()) posted push target 00000000128152f3 00000400:00000200:3.0:1775764333.445712:0:2384:0:(api-ni.c:2288:lnet_push_target_resize()) nbytes 272 success 00000400:00000010:3.0:1775764333.445715:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000eb71c03b. 00000400:00000010:3.0:1775764333.445720:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c18fc0e0. 00000400:00000010:3.0:1775764333.445722:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002eb1d61f. 00000400:00000010:3.0:1775764333.445730:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000002b7c1ce2. 00000400:00000010:3.0:1775764333.445731:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000de7b86da. 00000400:00000010:3.0:1775764333.445733:0:2384:0:(lib-cpt.c:1365:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004c0aef77. 00000400:00000200:3.0:1775764333.448076:0:2384:0:(peer.c:4224:lnet_peer_discovery_start()) discovery start: 0 00000100:00000010:3.0:1775764333.448192:0:2384:0:(ptlrpcd.c:749:ptlrpcd_init()) kmalloced '(ptlrpcds)': 16 at 000000007c30a68b. 00000400:00000200:2.0:1775764333.448362:0:2399:0:(peer.c:4050:lnet_peer_discovery()) started 00000100:00000010:3.0:1775764333.448647:0:2400:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000083b5d2ca. 00000020:00000010:3.0:1775764333.448653:0:2400:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006a3f1f36. 00000020:00000010:3.0:1775764333.448657:0:2400:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000222adc92. 00000020:00000010:3.0:1775764333.448660:0:2400:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000d2e3b121. 00000040:00000010:3.0:1775764333.448664:0:2400:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000d0233e5b. 00000020:00000010:3.0:1775764333.448669:0:2400:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000cc18c85b. 00000020:00000010:3.0:1775764333.448673:0:2400:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004a6aadb0. 00000100:00000010:2.0:1775764333.448719:0:2384:0:(ptlrpcd.c:846:ptlrpcd_init()) kmalloced '(pd)': 392 at 0000000079a29b0d. 00000100:00000010:2.0:1775764333.448725:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 00000000c503dc23. 00000100:00000010:2.0:1775764333.448728:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 000000002eebea62. 00000100:00000010:1.0:1775764333.451364:0:2401:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000cbe7e7ac. 00000020:00000010:1.0:1775764333.451386:0:2401:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ae1e386b. 00000020:00000010:1.0:1775764333.451393:0:2401:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000027a3d65b. 00000020:00000010:1.0:1775764333.451398:0:2401:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000060d317c5. 00000040:00000010:1.0:1775764333.451403:0:2401:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000e7743d86. 00000020:00000010:1.0:1775764333.451408:0:2401:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000012eb0654. 00000020:00000010:1.0:1775764333.451413:0:2401:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006d599192. 00000100:00000010:1.0:1775764333.452443:0:2402:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000097bf0978. 00000020:00000010:1.0:1775764333.452452:0:2402:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000027ae35f5. 00000020:00000010:1.0:1775764333.452456:0:2402:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000013a3591a. 00000020:00000010:1.0:1775764333.452460:0:2402:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000081e4d4e8. 00000040:00000010:1.0:1775764333.452465:0:2402:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000005ef7498d. 00000020:00000010:1.0:1775764333.452469:0:2402:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000b7e1275d. 00000020:00000010:1.0:1775764333.452473:0:2402:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007e0e68f2. 00000100:00000010:2.0:1775764333.452554:0:2384:0:(ptlrpcd.c:846:ptlrpcd_init()) kmalloced '(pd)': 392 at 0000000066a4a7de. 00000100:00000010:2.0:1775764333.452561:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 0000000077b3b2e1. 00000100:00000010:2.0:1775764333.452565:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 00000000151f685c. 00000100:00000010:2.0:1775764333.456907:0:2403:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000a3f23a95. 00000020:00000010:2.0:1775764333.456924:0:2403:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ec0a665c. 00000020:00000010:2.0:1775764333.456930:0:2403:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000413c1848. 00000020:00000010:2.0:1775764333.456934:0:2403:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000002d7a313e. 00000040:00000010:2.0:1775764333.456938:0:2403:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000037dcd7ac. 00000020:00000010:2.0:1775764333.456943:0:2403:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000d94fdb42. 00000020:00000010:2.0:1775764333.456948:0:2403:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000022fec89d. 00000100:00000010:3.0:1775764333.457620:0:2404:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000006cfa10a8. 00000020:00000010:3.0:1775764333.457626:0:2404:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000003eb9178. 00000020:00000010:3.0:1775764333.457630:0:2404:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b915e024. 00000020:00000010:3.0:1775764333.457632:0:2404:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c7b1e5b5. 00000040:00000010:3.0:1775764333.457634:0:2404:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000079797d63. 00000020:00000010:3.0:1775764333.457638:0:2404:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000043cdffcc. 00000020:00000010:3.0:1775764333.457642:0:2404:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e8513878. 00000100:00000010:2.0:1775764333.457949:0:2384:0:(connection.c:186:ptlrpc_connection_init()) kmalloced '(cpus_latency_qos)': 1088 at 00000000b4244603. 00000100:00000010:2.0:1775764333.458119:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000007a92ca98. 00000100:00000010:2.0:1775764333.458125:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 00000000f649849e. 00000100:00000010:2.0:1775764333.458127:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 0000000022ea04c8. 00000100:00000010:2.0:1775764333.458135:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000004e54386c. 00000100:00000010:2.0:1775764333.458138:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000007504b3d9. 00000100:00000010:2.0:1775764333.458140:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000001d5713e5. 00000001:00000010:2.0:1775764333.458346:0:2384:0:(nodemap_handler.c:4701:nodemap_config_alloc()) kmalloced '(config)': 480 at 000000001c89986c. 00000001:00000010:2.0:1775764333.458352:0:2384:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000bfaab419. 00000001:00000010:2.0:1775764333.458357:0:2384:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 8 at 00000000d7261b24. 00000001:00000010:2.0:1775764333.458361:0:2384:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 152 at 000000007ee50737. 00000001:00000010:2.0:1775764333.458372:0:2384:0:(nodemap_handler.c:3540:nodemap_create()) kmalloced '(nodemap)': 968 at 0000000098032890. 00000001:00000010:2.0:1775764333.458423:0:2384:0:(nodemap_lproc.c:1446:lprocfs_nodemap_register()) kmalloced '(nm_entry)': 48 at 0000000055faec7d. 02000000:00000010:1.0:1775764333.819825:0:2416:0:(upcall_cache.c:573:upcall_cache_init()) kmalloced '(cache)': 1208 at 00000000dcbe2f9a. 02000000:00000010:1.0:1775764333.819838:0:2416:0:(upcall_cache.c:581:upcall_cache_init()) kmalloced '(cache->uc_hashtable)': 1024 at 00000000382d11cf. 02000000:00000010:1.0:1775764333.819852:0:2416:0:(upcall_cache.c:573:upcall_cache_init()) kmalloced '(cache)': 1208 at 0000000051c2bbbc. 02000000:00000010:1.0:1775764333.819885:0:2416:0:(upcall_cache.c:581:upcall_cache_init()) kmalloced '(cache->uc_hashtable)': 16384 at 000000007ff6d993. 00000020:00000010:0.0:1775764336.124956:0:2469:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000c72d219f. 00000001:00000010:1.0:1775764336.461579:0:2482:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000003db8e144. 00000100:00000010:0.0:1775764336.461698:0:2482:0:(client.c:689:ptlrpc_init_rq_pool()) kmalloced '(pool)': 56 at 00000000eab8861f. 00000020:00000010:0.0:1775764336.461761:0:2482:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000173c1a17. 00000020:00000010:0.0:1775764336.461795:0:2482:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000baa2ee4d. 00000020:00000010:0.0:1775764336.461837:0:2482:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000baa2ee4d. 00000020:00000010:3.0:1775764336.875102:0:2495:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000aed2c1c8. 00000020:00000010:0.0:1775764337.125149:0:2508:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000efacb5d0. 00000020:00000010:3.0:1775764337.339658:0:2522:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000961e047d. 00008000:02000000:0.0:1775764337.622466:0:2535:0:(echo_client.c:2570:obdecho_init()) Echo OBD driver; http://www.lustre.org/ 00000020:00000010:0.0:1775764337.635716:0:2535:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000ed34404f. 00000020:00000010:0.0:1775764337.638141:0:2535:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000d5cf9722. 00000020:00000010:1.0:1775764338.218326:0:2558:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000009eac4fa6. 00000020:00000010:0.0:1775764341.566343:0:2904:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000004d422747. 00000020:00000010:0.0:1775764341.993574:0:2918:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000007bcf7a72. 00000020:00000010:3.0:1775764342.828341:0:2931:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 0000000060fbc391. 00000020:00000010:3.0:1775764342.828390:0:2931:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000005905c89e. 00000020:00000010:2.0:1775764343.347032:0:2952:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000009252b7c6. 00000020:00000010:2.0:1775764343.721570:0:2965:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000eb8bec36. 00000020:00000010:3.0:1775764343.727464:0:2965:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000007b07c97c. 00000020:00000010:1.0:1775764344.146991:0:2980:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000b5739b1d. 00000020:00000010:1.0:1775764344.147243:0:2980:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000005a37e7e8. 00000020:00000010:3.0:1775764345.179245:0:2993:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at 000000003e80261a. 00000020:00000010:3.0:1775764345.179258:0:2993:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001989da18. 00000020:00000010:3.0:1775764345.179263:0:2993:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b2c98435. 00000020:00000010:3.0:1775764345.179267:0:2993:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000002da576c0. 00000008:00000010:3.0:1775764345.179280:0:2993:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000054dcc75. 00020000:00000010:3.0:1775764345.179285:0:2993:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000038ea8b36. 00008000:00000010:3.0:1775764345.179364:0:2993:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000008fffc8f0. 00000080:00000010:3.0:1775764345.179378:0:2993:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000004987b814. 00000080:00000010:3.0:1775764345.179389:0:2993:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000258bd3e8. 00000020:00000010:3.0:1775764345.179397:0:2993:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a3bb5c94. 00000008:00000010:3.0:1775764345.179407:0:2993:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000b0760adb. 00020000:00000010:3.0:1775764345.179418:0:2993:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 0000000045c2d275. 00008000:00000010:3.0:1775764345.179433:0:2993:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 00000000a7be29af. 00000080:00000010:3.0:1775764345.179441:0:2993:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000cca33907. 00000080:00000010:2.0:1775764547.937442:0:6160:0:(super25.c:251:lustre_init_fs_context()) kmalloced '(lmd)': 128 at 00000000cb17b66d. 00000020:00000010:2.0:1775764547.937459:0:6160:0:(obd_mount.c:1364:lustre_parse_monolithic()) kmalloced '(lmd->lmd_params)': 4096 at 00000000fdd08ae4. 00000020:00000010:2.0:1775764547.937470:0:6160:0:(obd_mount.c:1566:lustre_parse_monolithic()) kmalloced '(lmd->lmd_dev)': 28 at 00000000c979a4de. 00000020:00000010:2.0:1775764547.937475:0:6160:0:(obd_mount.c:1622:lustre_parse_monolithic()) kmalloced '(lmd->lmd_profile)': 14 at 0000000019e1d1b0. 00000020:00000010:2.0:1775764547.937479:0:6160:0:(obd_mount.c:1650:lustre_parse_monolithic()) kmalloced '(lmd->lmd_opts)': 17 at 000000008f99b1b6. 00000020:00000010:3.0:1775764547.937976:0:6160:0:(obd_mount.c:632:lustre_init_lsi()) kmalloced '(lsi)': 1880 at 00000000f8842aa1. 00000020:00000010:3.0:1775764547.938002:0:6160:0:(obd_mount.c:323:lustre_start_mgc()) kmalloced '(mgcname)': 23 at 0000000079c95200. 00000020:00000010:3.0:1775764547.938006:0:6160:0:(obd_mount.c:331:lustre_start_mgc()) kmalloced '(data)': 192 at 00000000022c6efc. 00000100:00000010:3.0:1775764547.938014:0:6160:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 0000000046f50674. 00000400:00000200:3.0:1775764547.938018:0:6160:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1775764547.938023:0:6160:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:3.0:1775764547.938032:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764547.938039:0:6160:0:(peer.c:229:lnet_peer_alloc()) alloc '(lp)': 376 at 0000000006eda7a3. 00000400:00000200:3.0:1775764547.938042:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764547.938044:0:6160:0:(peer.c:262:lnet_peer_alloc()) 0000000006eda7a3 nid 192.168.202.104@tcp 00000400:00000010:3.0:1775764547.938048:0:6160:0:(peer.c:193:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 00000000fd0ecf56. 00000400:00000200:3.0:1775764547.938050:0:6160:0:(peer.c:202:lnet_peer_net_alloc()) 00000000fd0ecf56 net tcp 00000400:00000200:3.0:1775764547.938052:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764547.938055:0:6160:0:(peer.c:137:lnet_peer_ni_alloc()) alloc '(lpni)': 384 at 00000000f1da7899. 00000400:00000200:3.0:1775764547.938058:0:6160:0:(peer.c:183:lnet_peer_ni_alloc()) 00000000f1da7899 nid 192.168.202.104@tcp 00000400:00000200:3.0:1775764547.938066:0:6160:0:(peer.c:1091:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.202.104@tcp: -2 00000400:00000200:3.0:1775764547.938072:0:6160:0:(peer.c:1648:lnet_peer_attach_peer_ni()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp flags 0x100001 00000400:00000200:3.0:1775764547.938077:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764547.938144:0:6160:0:(peer.c:2414:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.104@tcp: 0 00000400:00000200:3.0:1775764547.938147:0:6160:0:(peer.c:2736:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:3.0:1775764547.938149:0:6160:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1775764547.938152:0:6160:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000020:00000010:3.0:1775764547.938157:0:6160:0:(obd_mount.c:463:lustre_start_mgc()) kmalloced '(uuid)': 40 at 0000000090cbab9b. 00000020:00000010:3.0:1775764547.938186:0:6160:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6592 at 000000008bcfeea1. 00000020:00000010:3.0:1775764547.938193:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 0000000083e9be91. 00000020:00000010:3.0:1775764547.939733:0:6160:0:(obd_mount.c:240:lustre_start_simple()) kmalloced '(lcfg)': 104 at 00000000d0b619fb. 00000001:00000010:3.0:1775764547.939774:0:6160:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000c5714e6e. 00000001:00000010:3.0:1775764547.939782:0:6160:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000b10547d7. 00000001:00000010:3.0:1775764547.939787:0:6160:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000006de3435f. 00000001:00000010:3.0:1775764547.942289:0:6160:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000008a5c7681. 00000001:00000010:3.0:1775764547.942293:0:6160:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000007f9da185. 00000001:00000010:3.0:1775764547.942296:0:6160:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000006f462db2. 00000020:00000010:3.0:1775764547.942303:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e07977ad. 00000020:00000010:3.0:1775764547.942310:0:6160:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000001230894f. 00000020:00000010:3.0:1775764547.942343:0:6160:0:(tgt_main.c:757:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000067181d6f. 00000008:00000010:3.0:1775764547.942351:0:6160:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000a021b43c. 00020000:00000010:3.0:1775764547.942357:0:6160:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000003da9f5d9. 00008000:00000010:3.0:1775764547.942363:0:6160:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000004d2cc17. 00000080:00000010:3.0:1775764547.942370:0:6160:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000b5e74327. 00000020:00000010:3.0:1775764547.942376:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c690e0af. 00000020:00000010:3.0:1775764547.942379:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000d19d4c82. 00000040:00000010:3.0:1775764547.942383:0:6160:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000095e8b699. 00000020:00000010:3.0:1775764547.942387:0:6160:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000097be601e. 00000020:00000010:3.0:1775764547.942392:0:6160:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000001fae4d25. 00040000:00000010:3.0:1775764547.942396:0:6160:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000063523f6d. 00040000:00000010:3.0:1775764547.942402:0:6160:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000b5d79eeb. 10000000:00000010:3.0:1775764547.942409:0:6160:0:(mgc_request.c:852:mgc_device_alloc()) kmalloced '(lu)': 64 at 000000004a50f8a8. 00010000:00000010:3.0:1775764547.942432:0:6160:0:(ldlm_lib.c:420:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 000000000eade53e. 00010000:00000010:3.0:1775764547.942439:0:6160:0:(ldlm_lib.c:433:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000008b17a25e. 00010000:00000010:3.0:1775764547.942447:0:6160:0:(ldlm_lockd.c:3356:ldlm_setup()) kmalloced '(ldlm_state)': 32 at 00000000a64bccb5. 00000100:00000010:3.0:1775764547.942612:0:6160:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at 00000000312d27db. 00000100:00000010:3.0:1775764547.942621:0:6160:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 00000000ea785009. 00000100:00000010:3.0:1775764547.942640:0:6160:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 00000000016d01c8. 00000100:00000010:3.0:1775764547.942649:0:6160:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 000000003c335ea6. 00000100:00000010:3.0:1775764547.942658:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bd326cda. 00000100:00000010:3.0:1775764547.942664:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000038c33d42. 00000100:00000010:3.0:1775764547.942669:0:6160:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 00000000abecbe63. 00000100:00000010:3.0:1775764547.942683:0:6160:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 00000000296c0cc1. 00000100:00000010:3.0:1775764547.942690:0:6160:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 0000000081a9561f. 00000100:00000010:3.0:1775764547.942694:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002d69effd. 00000100:00000010:3.0:1775764547.942703:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000171d6d2a. 00000400:00000200:3.0:1775764547.942709:0:6160:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 15 lazy 00000020:00000010:3.0:1775764547.942755:0:6160:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000043cacf08. 00000020:00000010:3.0:1775764547.942761:0:6160:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 000000005900f957. 00000400:00000200:2.0:1775764547.942964:0:2399:0:(peer.c:3997:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1775764547.942974:0:2399:0:(peer.c:4109:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100041 00000400:00000010:2.0:1775764547.942984:0:2399:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ea4f13ad. 00000400:00000010:2.0:1775764547.943006:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007682e321. 00000400:00000010:2.0:1775764547.943030:0:2399:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at 00000000ef013e55. 00000400:00000200:2.0:1775764547.943035:0:2399:0:(lib-move.c:5564:LNetGet()) LNetGet -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764547.943049:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764547.943056:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764547.943063:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764547.943069:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764547.943073:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764547.943076:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764547.943081:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [1:1:256:4294967295] 192.168.202.104@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1775764547.943085:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764547.943092:0:2399:0:(lib-move.c:2011:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.104@tcp 00000400:00000200:2.0:1775764547.943099:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : GET try# 0 00000800:00000200:2.0:1775764547.943112:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764547.943116:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000f221f19c. 00000400:00000200:2.0:1775764547.943126:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000010:2.0:1775764547.943130:0:2399:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000001f4dd10c. 00000800:00000010:2.0:1775764547.943134:0:2399:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000aabe49c9. 00000800:00000200:2.0:1775764547.943140:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (2) 00000800:00000200:2.1F:1775764547.943145:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (2) 00000400:00000200:2.0:1775764547.943160:0:2399:0:(peer.c:3797:lnet_peer_send_ping()) peer 192.168.202.104@tcp 00000400:00000200:2.0:1775764547.943163:0:2399:0:(peer.c:4127:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100241 rc 0 00000100:00000010:3.0:1775764547.946518:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000cb277bea. 00000100:00000010:3.0:1775764547.946538:0:6160:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 0000000078f220d3. 00000100:00000010:3.0:1775764547.946542:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000007a8ae8b4. 00000100:00000010:3.0:1775764547.946547:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000529ae55e. 00000100:00000010:3.0:1775764547.946550:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000479a74d7. 00000100:00000010:3.0:1775764547.946554:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000fc86924b. 00000100:00000010:3.0:1775764547.946556:0:6160:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 0000000059e56e1b. 00000100:00000010:3.0:1775764547.946558:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000043830648. 00000100:00000010:3.0:1775764547.946560:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000088e42d75. 00000100:00000010:3.0:1775764547.946562:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000005d9c727d. 00000100:00000200:3.0:1775764547.946598:0:6160:0:(service.c:848:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15 00000100:00000010:3.0:1775764547.946604:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000dd7b3928. 00000800:00000010:0.0:1775764547.947816:0:2388:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 00000000a151ae1f. 00000800:00000010:0.0:1775764547.947833:0:2388:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000007e98606e. 00000800:00000010:0.0:1775764547.947843:0:2388:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000e5d99874. 00000800:00000010:0.0:1775764547.947868:0:2388:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000e5d99874. 00000100:00000010:0.0:1775764547.947906:0:6161:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000e5d99874. 00000020:00000010:0.0:1775764547.947911:0:6161:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000007ed7bbd0. 00000020:00000010:0.0:1775764547.947919:0:6161:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcf6e424. 00000020:00000010:0.0:1775764547.947923:0:6161:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000f05302b4. 00000040:00000010:0.0:1775764547.947927:0:6161:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000015e97ecb. 00000020:00000010:0.0:1775764547.947933:0:6161:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000027e2a787. 00000020:00000010:0.0:1775764547.947953:0:6161:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000002326e9fd. 00000020:00000010:0.0:1775764547.947961:0:6161:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000547f39ee. 80000000:00000010:0.0:1775764547.947966:0:6161:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000e928a179. 40000000:00000010:0.0:1775764547.947971:0:6161:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000177639e1. 00100000:00000010:0.0:1775764547.947979:0:6161:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000002862c9eb. 00040000:00000010:0.0:1775764547.947984:0:6161:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000004901cf2b. 00040000:00000010:0.0:1775764547.947990:0:6161:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000003051e03. 00040000:00000010:0.0:1775764547.947995:0:6161:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000061dc9eeb. 00000004:00000010:0.0:1775764547.948000:0:6161:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000002e39b344. 00000004:00000010:0.0:1775764547.948006:0:6161:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000ebf45e9a. 00000004:00000010:0.0:1775764547.948012:0:6161:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000634b93f6. 00002000:00000010:0.0:1775764547.948018:0:6161:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000c51a2e75. 00000004:00000010:0.0:1775764547.948024:0:6161:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000517db740. 00000100:00000200:0.0:1775764547.948031:0:6161:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:0.0:1775764547.948041:0:6161:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000eedaeaa7. 00000400:00000010:0.0:1775764547.948049:0:6161:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005ecd3238. 00000100:00000010:0.0:1775764547.948072:0:6161:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 000000000283d935. 00000100:00000200:0.0:1775764547.948094:0:6161:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:2.0:1775764547.948535:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000c5d2aa4b. 00000100:00000010:0.0:1775764547.953297:0:6162:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000283cd0bf. 00000020:00000010:0.0:1775764547.953310:0:6162:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000fc233615. 00000020:00000010:0.0:1775764547.953319:0:6162:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009c16d323. 00000020:00000010:0.0:1775764547.953323:0:6162:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008d6923e3. 00000040:00000010:0.0:1775764547.953398:0:6162:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000032da3fcd. 00000020:00000010:0.0:1775764547.953406:0:6162:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000057b83bd3. 00000020:00000010:0.0:1775764547.953413:0:6162:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000004934ea9f. 00000020:00000010:0.0:1775764547.953419:0:6162:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000009b76ff79. 80000000:00000010:0.0:1775764547.953424:0:6162:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000f231fb8c. 40000000:00000010:0.0:1775764547.953429:0:6162:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 0000000006711dc2. 00100000:00000010:0.0:1775764547.953436:0:6162:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000c6229b5d. 00040000:00000010:0.0:1775764547.953440:0:6162:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000000f52e3f4. 00040000:00000010:0.0:1775764547.953446:0:6162:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000003aa32670. 00040000:00000010:0.0:1775764547.953450:0:6162:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000003948af8e. 00000004:00000010:0.0:1775764547.953456:0:6162:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000ab223851. 00000004:00000010:0.0:1775764547.953462:0:6162:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 000000001e419e9a. 00000004:00000010:0.0:1775764547.953466:0:6162:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000b72f4a62. 00002000:00000010:0.0:1775764547.953471:0:6162:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000007daa95bf. 00000004:00000010:0.0:1775764547.953476:0:6162:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000f0e97e13. 00000100:00000010:0.0:1775764547.953482:0:6162:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000024d47e98. 00000100:00000200:0.0:1775764547.953587:0:6162:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:3.0:1775764547.954253:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 000000004d079803. 00000400:00000200:0.0:1775764547.954776:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:0.1F:1775764547.954784:0:2388:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:0.1:1775764547.954826:0:2388:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.1:1775764547.954831:0:2388:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000f221f19c type 2, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1775764547.954836:0:2388:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.202.104@tcp p 3.x 192.168.202.4 -> 192.168.202.104:988 incarnation:1775764356495133599 sched[0] 00000800:00000010:0.0:1775764547.954840:0:2388:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 000000007e98606e. 00000400:00000200:0.0:1775764547.954937:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.202.104:988 00000100:00000010:3.0:1775764547.955370:0:6163:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000008ca138a0. 00000020:00000010:3.0:1775764547.955376:0:6163:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000f889a781. 00000020:00000010:3.0:1775764547.955382:0:6163:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000071773679. 00000020:00000010:3.0:1775764547.955385:0:6163:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000003fd90b29. 00000040:00000010:3.0:1775764547.955389:0:6163:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000001d2dbdc5. 00000020:00000010:3.0:1775764547.955393:0:6163:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000032ed0e38. 00000020:00000010:3.0:1775764547.955412:0:6163:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000008bd8bf1d. 00000020:00000010:3.0:1775764547.955418:0:6163:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000007cdd11d0. 80000000:00000010:3.0:1775764547.955422:0:6163:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000380e4e22. 40000000:00000010:3.0:1775764547.955425:0:6163:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000e590d0cd. 00100000:00000010:3.0:1775764547.955429:0:6163:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 0000000045f79256. 00040000:00000010:3.0:1775764547.955433:0:6163:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000b1b3f20d. 00040000:00000010:3.0:1775764547.955437:0:6163:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000005c655451. 00040000:00000010:3.0:1775764547.955441:0:6163:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000009666e6f8. 00000004:00000010:3.0:1775764547.955445:0:6163:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000088651576. 00000004:00000010:3.0:1775764547.955450:0:6163:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000d760ff52. 00000004:00000010:3.0:1775764547.955459:0:6163:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000e2e4fbce. 00002000:00000010:3.0:1775764547.955463:0:6163:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000706eac9e. 00000004:00000010:3.0:1775764547.955467:0:6163:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000fbd5e04e. 00000100:00000200:3.0:1775764547.955472:0:6163:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:3.0:1775764547.955480:0:6163:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000094de006. 00000400:00000010:3.0:1775764547.955524:0:6163:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d235a5ec. 00000100:00000010:3.0:1775764547.955536:0:6163:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000fa58e7ef. 00000100:00000200:3.0:1775764547.955548:0:6163:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:2.0:1775764547.955603:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 000000005c68f838. 00000800:00000200:0.0:1775764547.956116:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764547.956121:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000f221f19c. 00000400:00000200:0.0:1775764547.956127:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764547.956136:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: GET: OK 00000400:00000200:0.0:1775764547.956144:0:2393:0:(peer.c:3090:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1775764547.956147:0:2393:0:(peer.c:3042:lnet_discovery_event_send()) Ping Send to 192.168.202.104@tcp: 0 00000100:00000010:3.0:1775764547.958420:0:6164:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000006960390a. 00000020:00000010:3.0:1775764547.958430:0:6164:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000068d3c2bb. 00000020:00000010:3.0:1775764547.958437:0:6164:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001482b5a7. 00000020:00000010:3.0:1775764547.958442:0:6164:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c2766df8. 00000040:00000010:3.0:1775764547.958447:0:6164:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000e6d04d8e. 00000020:00000010:3.0:1775764547.958452:0:6164:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000002232fbb0. 00000020:00000010:3.0:1775764547.958457:0:6164:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000fce02ad8. 00000020:00000010:3.0:1775764547.958464:0:6164:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000d7b494dc. 80000000:00000010:3.0:1775764547.958469:0:6164:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000e0da44f9. 40000000:00000010:3.0:1775764547.958473:0:6164:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000905bf040. 00100000:00000010:3.0:1775764547.958478:0:6164:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000a6c29191. 00040000:00000010:3.0:1775764547.958482:0:6164:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000d4f2e803. 00040000:00000010:3.0:1775764547.958512:0:6164:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000eef68a9e. 00040000:00000010:3.0:1775764547.958516:0:6164:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000765a5e56. 00000004:00000010:3.0:1775764547.958521:0:6164:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000006b885479. 00000004:00000010:3.0:1775764547.958527:0:6164:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 000000008ab819a8. 00000004:00000010:3.0:1775764547.958532:0:6164:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000d61ff38a. 00002000:00000010:3.0:1775764547.958537:0:6164:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000a128fbb8. 00000004:00000010:3.0:1775764547.958542:0:6164:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000af5c78bf. 00000100:00000010:3.0:1775764547.958547:0:6164:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000777f5754. 00000100:00000200:3.0:1775764547.958569:0:6164:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:2.0:1775764547.958640:0:6160:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at 0000000016943886. 00000100:00000010:2.0:1775764547.958648:0:6160:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 00000000329ffa20. 00000100:00000010:2.0:1775764547.958681:0:6160:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 00000000c328c865. 00000100:00000010:2.0:1775764547.958690:0:6160:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 000000007065a374. 00000100:00000010:2.0:1775764547.958695:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000516c941a. 00000100:00000010:2.0:1775764547.958701:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a0d3e949. 00000100:00000010:2.0:1775764547.958703:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000012fae94. 00000100:00000010:2.0:1775764547.958707:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000830dab95. 00000100:00000010:2.0:1775764547.958709:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006e018df7. 00000100:00000010:2.0:1775764547.958720:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000043f0880e. 00000100:00000010:2.0:1775764547.958723:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000008f66588. 00000100:00000010:2.0:1775764547.958727:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000000425e45c. 00000100:00000010:2.0:1775764547.958729:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002d185f96. 00000100:00000010:2.0:1775764547.958732:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d396009b. 00000100:00000010:2.0:1775764547.958735:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d36c1ade. 00000100:00000010:2.0:1775764547.958738:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000797ed71f. 00000100:00000010:2.0:1775764547.958740:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000633a0568. 00000100:00000010:2.0:1775764547.958749:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005e9ab7a9. 00000100:00000010:2.0:1775764547.958753:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000dc28c1cf. 00000100:00000010:2.0:1775764547.958756:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009b441742. 00000100:00000010:2.0:1775764547.958758:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000285951b3. 00000100:00000010:2.0:1775764547.958767:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000043ff4cc2. 00000100:00000010:2.0:1775764547.958769:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000036bc8084. 00000100:00000010:2.0:1775764547.958773:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a179a29a. 00000100:00000010:2.0:1775764547.958775:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000018e269b6. 00000100:00000010:2.0:1775764547.958783:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006ba4b12a. 00000100:00000010:2.0:1775764547.958785:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007efab4be. 00000100:00000010:2.0:1775764547.958789:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000671e9a2c. 00000100:00000010:2.0:1775764547.958791:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006c3c586d. 00000100:00000010:2.0:1775764547.958801:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fbfdca8e. 00000100:00000010:2.0:1775764547.958804:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a545c1e1. 00000100:00000010:2.0:1775764547.958807:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006c763e02. 00000100:00000010:2.0:1775764547.958810:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000046518b33. 00000100:00000010:2.0:1775764547.958820:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001b5b8966. 00000100:00000010:2.0:1775764547.958824:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b1dc86a4. 00000100:00000010:2.0:1775764547.958828:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f9d9b8ed. 00000100:00000010:2.0:1775764547.958831:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000921004f2. 00000100:00000010:2.0:1775764547.958835:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000cbbd66b5. 00000100:00000010:2.0:1775764547.958837:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003c6da84e. 00000100:00000010:2.0:1775764547.958841:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005ae35d0e. 00000100:00000010:2.0:1775764547.958843:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007383dd08. 00000100:00000010:2.0:1775764547.958852:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e908a311. 00000100:00000010:2.0:1775764547.958856:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000081262de5. 00000100:00000010:2.0:1775764547.958869:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000060e610bf. 00000100:00000010:2.0:1775764547.958872:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a1828a7f. 00000100:00000010:2.0:1775764547.958876:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d26f6406. 00000100:00000010:2.0:1775764547.958879:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004d630f80. 00000100:00000010:2.0:1775764547.958882:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f9df8fe0. 00000100:00000010:2.0:1775764547.958885:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000997adc9b. 00000100:00000010:2.0:1775764547.958895:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004460e0bb. 00000100:00000010:2.0:1775764547.958898:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000de95003e. 00000100:00000010:2.0:1775764547.958901:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c80ca2c7. 00000100:00000010:2.0:1775764547.958908:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c0c20b6f. 00000100:00000010:2.0:1775764547.958911:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ce4ff23e. 00000100:00000010:2.0:1775764547.958914:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a07a5fbb. 00000100:00000010:2.0:1775764547.958918:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000000c601e08. 00000100:00000010:2.0:1775764547.958920:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000076f5c526. 00000100:00000010:2.0:1775764547.958929:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000043661975. 00000100:00000010:2.0:1775764547.958933:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009ee7fee7. 00000100:00000010:2.0:1775764547.958937:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007065a4b1. 00000100:00000010:2.0:1775764547.958940:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e1c5e714. 00000100:00000010:2.0:1775764547.958943:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000005898ba3. 00000100:00000010:2.0:1775764547.958946:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001b61c25a. 00000100:00000010:2.0:1775764547.958950:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000037cf0068. 00000100:00000010:2.0:1775764547.958953:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000868d5805. 00000100:00000010:2.0:1775764547.958964:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d7556ed6. 00000100:00000010:2.0:1775764547.958967:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006aa313b9. 00000100:00000010:2.0:1775764547.958972:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000031fcc5f5. 00000100:00000010:2.0:1775764547.958975:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000dd7d4621. 00000100:00000010:2.0:1775764547.958980:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000056b9e6b8. 00000100:00000010:2.0:1775764547.958982:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d93c099a. 00000100:00000010:2.0:1775764547.958987:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000044dc3e44. 00000100:00000010:2.0:1775764547.958990:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000005e25d411. 00000100:00000010:2.0:1775764547.958999:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000003fe4d6e2. 00000100:00000010:2.0:1775764547.959002:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d4c40d75. 00000100:00000010:2.0:1775764547.959006:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000033d02c82. 00000100:00000010:2.0:1775764547.959009:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003c83d0c7. 00000100:00000010:2.0:1775764547.959012:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a462e76a. 00000100:00000010:2.0:1775764547.959015:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000088d3e957. 00000100:00000010:2.0:1775764547.959018:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000305f3817. 00000100:00000010:2.0:1775764547.959021:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000018c7f536. 00000100:00000010:2.0:1775764547.959031:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004a979b59. 00000100:00000010:2.0:1775764547.959034:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b1e35a04. 00000100:00000010:2.0:1775764547.959043:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009170fe18. 00000100:00000010:2.0:1775764547.959046:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000005a79a596. 00000100:00000010:2.0:1775764547.959051:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000050ed4ed2. 00000100:00000010:2.0:1775764547.959054:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f311222b. 00000100:00000010:2.0:1775764547.959059:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b49da0bb. 00000100:00000010:2.0:1775764547.959062:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000030b0e7de. 00000100:00000010:2.0:1775764547.959071:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000077d5d985. 00000100:00000010:2.0:1775764547.959075:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f070cad3. 00000100:00000010:2.0:1775764547.959078:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a7b139ee. 00000100:00000010:2.0:1775764547.959081:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bbb0943d. 00000100:00000010:2.0:1775764547.959085:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002594370d. 00000100:00000010:2.0:1775764547.959087:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fc85ba51. 00000100:00000010:2.0:1775764547.959091:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000000ce2cbb6. 00000100:00000010:2.0:1775764547.959093:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000851dab7f. 00000100:00000010:2.0:1775764547.959103:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000bc37a6d7. 00000100:00000010:2.0:1775764547.959106:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cbcd0d8e. 00000100:00000010:2.0:1775764547.959111:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000061dac068. 00000100:00000010:2.0:1775764547.959115:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a898a27a. 00000100:00000010:2.0:1775764547.959119:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006f81aed2. 00000100:00000010:2.0:1775764547.959122:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e0289470. 00000100:00000010:2.0:1775764547.959126:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000011ab4a5. 00000100:00000010:2.0:1775764547.959129:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ce45d8e0. 00000100:00000010:2.0:1775764547.959137:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002f48e894. 00000100:00000010:2.0:1775764547.959141:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b8a36f2e. 00000100:00000010:2.0:1775764547.959144:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000490c4170. 00000100:00000010:2.0:1775764547.959148:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004eda8e7e. 00000100:00000010:2.0:1775764547.959151:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008b9dfa39. 00000100:00000010:2.0:1775764547.959154:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d2f70dd5. 00000100:00000010:2.0:1775764547.959158:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000eaf99dc4. 00000100:00000010:2.0:1775764547.959161:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000008806aa3. 00000100:00000010:2.0:1775764547.959170:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ff197be8. 00000100:00000010:2.0:1775764547.959177:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000015e84bf1. 00000100:00000010:2.0:1775764547.959180:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000051cb0c0c. 00000100:00000010:2.0:1775764547.959182:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cac537f3. 00000100:00000010:2.0:1775764547.959186:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004bcd364a. 00000100:00000010:2.0:1775764547.959187:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c557373c. 00000100:00000010:2.0:1775764547.959191:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f419d9c8. 00000100:00000010:2.0:1775764547.959193:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000178d5bcf. 00000100:00000010:2.0:1775764547.959204:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001fe11ce0. 00000100:00000010:2.0:1775764547.959207:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000255b43cd. 00000100:00000010:2.0:1775764547.959212:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b2cb2e4d. 00000100:00000010:2.0:1775764547.959214:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000074ad862b. 00000100:00000010:2.0:1775764547.959219:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005e032003. 00000100:00000010:2.0:1775764547.959228:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001cac1319. 00000100:00000010:2.0:1775764547.959234:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000044264daf. 00000100:00000010:2.0:1775764547.959236:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000eae64d54. 00000100:00000010:2.0:1775764547.959245:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005a572597. 00000100:00000010:2.0:1775764547.959248:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000278a71c6. 00000100:00000010:2.0:1775764547.959252:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007604913d. 00000100:00000010:2.0:1775764547.959255:0:6160:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 0000000013e524c5. 00000100:00000010:2.0:1775764547.959268:0:6160:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 0000000054dbc65c. 00000100:00000010:2.0:1775764547.959276:0:6160:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 0000000038937e2b. 00000100:00000010:2.0:1775764547.959280:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000005059c411. 00000100:00000010:2.0:1775764547.959283:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d7cb5acc. 00000100:00000010:2.0:1775764547.959286:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ecdf79b5. 00000100:00000010:2.0:1775764547.959289:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000107bc900. 00000100:00000010:2.0:1775764547.959292:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b28a0f8f. 00000100:00000010:2.0:1775764547.959301:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000433b4d99. 00000100:00000010:2.0:1775764547.959304:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b5c67453. 00000100:00000010:2.0:1775764547.959308:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000019f322af. 00000100:00000010:2.0:1775764547.959310:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000269c51b5. 00000100:00000010:2.0:1775764547.959313:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000491f7784. 00000100:00000010:2.0:1775764547.959320:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002e89c50c. 00000100:00000010:2.0:1775764547.959324:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001c1d0b32. 00000100:00000010:2.0:1775764547.959326:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002a39b60f. 00000100:00000010:2.0:1775764547.959335:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000875180bd. 00000100:00000010:2.0:1775764547.959338:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000004ec6c2c. 00000100:00000010:2.0:1775764547.959341:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008d48ac5e. 00000100:00000010:2.0:1775764547.959344:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000825713ce. 00000100:00000010:2.0:1775764547.959348:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000044991a26. 00000100:00000010:2.0:1775764547.959350:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000098930bf2. 00000100:00000010:2.0:1775764547.959354:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000081d10737. 00000100:00000010:2.0:1775764547.959356:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000aadb2189. 00000100:00000010:2.0:1775764547.959365:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000061e0029e. 00000100:00000010:2.0:1775764547.959368:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002f63454c. 00000100:00000010:2.0:1775764547.959372:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000363c027c. 00000100:00000010:2.0:1775764547.959380:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f2de6793. 00000100:00000010:2.0:1775764547.959384:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008171df30. 00000100:00000010:2.0:1775764547.959386:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000602409c5. 00000100:00000010:2.0:1775764547.959390:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f7ac53c0. 00000100:00000010:2.0:1775764547.959393:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d087dd24. 00000100:00000010:2.0:1775764547.959402:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000066137806. 00000100:00000010:2.0:1775764547.959405:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009a8a3a9f. 00000100:00000010:2.0:1775764547.959408:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000009620556. 00000100:00000010:2.0:1775764547.959411:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cf3be38f. 00000100:00000010:2.0:1775764547.959414:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e91b5eb6. 00000100:00000010:2.0:1775764547.959416:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000652c2dad. 00000100:00000010:2.0:1775764547.959420:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c5d3004a. 00000100:00000010:2.0:1775764547.959422:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e9776aa5. 00000100:00000010:2.0:1775764547.959432:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000027e94c16. 00000100:00000010:2.0:1775764547.959435:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000163abb66. 00000100:00000010:2.0:1775764547.959439:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000482334a0. 00000100:00000010:2.0:1775764547.959441:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000077d0208a. 00000100:00000010:2.0:1775764547.959449:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fb3c4c0d. 00000100:00000010:2.0:1775764547.959452:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000dcd396e0. 00000100:00000010:2.0:1775764547.959455:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007978ed0d. 00000100:00000010:2.0:1775764547.959457:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ee25f586. 00000100:00000010:2.0:1775764547.959466:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c15b518c. 00000100:00000010:2.0:1775764547.959469:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000094c0f6e7. 00000100:00000010:2.0:1775764547.959472:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e7e48d80. 00000100:00000010:2.0:1775764547.959475:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000650e92d0. 00000100:00000010:2.0:1775764547.959479:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006c3d7dc6. 00000100:00000010:2.0:1775764547.959481:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006c62fab7. 00000100:00000010:2.0:1775764547.959523:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a50453af. 00000100:00000010:2.0:1775764547.959527:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001c37ed9d. 00000100:00000010:2.0:1775764547.959537:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006eea6ea4. 00000100:00000010:2.0:1775764547.959540:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009cc6ffd8. 00000100:00000010:2.0:1775764547.959543:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ab50dafe. 00000100:00000010:2.0:1775764547.959552:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006ea20f95. 00000100:00000010:2.0:1775764547.959556:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a3ecb3fb. 00000100:00000010:2.0:1775764547.959559:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f0773c9e. 00000100:00000010:2.0:1775764547.959562:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ac590640. 00000100:00000010:2.0:1775764547.959564:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000136a8700. 00000100:00000010:2.0:1775764547.959575:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000213c701b. 00000100:00000010:2.0:1775764547.959578:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000dc8fe8da. 00000100:00000010:2.0:1775764547.959582:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000832c4e73. 00000100:00000010:2.0:1775764547.959585:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000044883043. 00000100:00000010:2.0:1775764547.959589:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000aad77fd9. 00000100:00000010:2.0:1775764547.959592:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004c590611. 00000100:00000010:2.0:1775764547.959596:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000043b5a72. 00000100:00000010:2.0:1775764547.959599:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007fc35d34. 00000100:00000010:2.0:1775764547.959608:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000034e65834. 00000100:00000010:2.0:1775764547.959611:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004f8fab2a. 00000100:00000010:2.0:1775764547.959614:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b82432e3. 00000100:00000010:2.0:1775764547.959621:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000be6655b5. 00000100:00000010:2.0:1775764547.959625:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000efe4f5be. 00000100:00000010:2.0:1775764547.959627:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f45e7ebe. 00000100:00000010:2.0:1775764547.959631:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ac77ffc8. 00000100:00000010:2.0:1775764547.959633:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001aa1af90. 00000100:00000010:2.0:1775764547.959644:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000acd7188e. 00000100:00000010:2.0:1775764547.959647:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007aaf54cc. 00000100:00000010:2.0:1775764547.959652:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000626d41e3. 00000100:00000010:2.0:1775764547.959654:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007780834c. 00000100:00000010:2.0:1775764547.959659:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008697a34a. 00000100:00000010:2.0:1775764547.959662:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000007baeada. 00000100:00000010:2.0:1775764547.959666:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000009a48b7f. 00000100:00000010:2.0:1775764547.959668:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b384d705. 00000100:00000010:2.0:1775764547.959678:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000096b77e26. 00000100:00000010:2.0:1775764547.959681:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e8e76ee4. 00000100:00000010:2.0:1775764547.959685:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000199d256a. 00000100:00000010:2.0:1775764547.959695:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ff2a822e. 00000100:00000010:2.0:1775764547.959699:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000052a092ac. 00000100:00000010:2.0:1775764547.959702:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000eb0f9616. 00000100:00000010:2.0:1775764547.959706:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000032067e27. 00000100:00000010:2.0:1775764547.959708:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cbaa84db. 00000100:00000010:2.0:1775764547.959717:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000723a9f19. 00000100:00000010:2.0:1775764547.959720:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e70efdba. 00000100:00000010:2.0:1775764547.959724:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004398174b. 00000100:00000010:2.0:1775764547.959726:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003edac75e. 00000100:00000010:2.0:1775764547.959730:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e200042d. 00000100:00000010:2.0:1775764547.959733:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000000d85d3e6. 00000100:00000010:2.0:1775764547.959737:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001d2af29e. 00000100:00000010:2.0:1775764547.959739:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f02eeaf5. 00000100:00000010:2.0:1775764547.959749:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fb154f18. 00000100:00000010:2.0:1775764547.959753:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004699c4c6. 00000100:00000010:2.0:1775764547.959761:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001ebea789. 00000100:00000010:2.0:1775764547.959763:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d66a0f13. 00000100:00000010:2.0:1775764547.959767:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009569efbe. 00000100:00000010:2.0:1775764547.959769:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000008235161. 00000100:00000010:2.0:1775764547.959773:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f85710d6. 00000100:00000010:2.0:1775764547.959775:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000dc38e655. 00000100:00000010:2.0:1775764547.959785:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008c94b759. 00000100:00000010:2.0:1775764547.959788:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000743ae143. 00000100:00000010:2.0:1775764547.959792:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f0804952. 00000100:00000010:2.0:1775764547.959794:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004f8469eb. 00000100:00000010:2.0:1775764547.959798:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b0c75ec5. 00000100:00000010:2.0:1775764547.959800:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ab693acd. 00000100:00000010:2.0:1775764547.959804:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000837eeef2. 00000100:00000010:2.0:1775764547.959806:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f2eb3191. 00000100:00000010:2.0:1775764547.959814:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000221d02d8. 00000100:00000010:2.0:1775764547.959817:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a3abbfc1. 00000100:00000010:2.0:1775764547.959821:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001de7e7af. 00000100:00000010:2.0:1775764547.959831:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000891c28e6. 00000100:00000010:2.0:1775764547.959836:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a38794a1. 00000100:00000010:2.0:1775764547.959838:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f64a8710. 00000100:00000010:2.0:1775764547.959842:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002c22a4ca. 00000100:00000010:2.0:1775764547.959844:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000017a73a3e. 00000100:00000010:2.0:1775764547.959854:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001fb7bafb. 00000100:00000010:2.0:1775764547.959857:0:6160:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c198ffd2. 00000100:00000010:2.0:1775764547.959872:0:6160:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000171b79ee. 00000400:00000200:2.0:1775764547.959877:0:6160:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 17 lazy 00000020:00000010:2.0:1775764547.959955:0:6160:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000048053f7c. 00000020:00000010:2.0:1775764547.959962:0:6160:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 00000000811af0fc. 00000100:00000010:2.0:1775764547.960020:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000002c10b9b2. 00000100:00000010:2.0:1775764547.960028:0:6160:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 0000000052c19b1a. 00000100:00000010:2.0:1775764547.960032:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000036b9175e. 00000100:00000010:2.0:1775764547.960037:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000b7bfd1cd. 00000100:00000010:2.0:1775764547.960045:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000063a28d2b. 00000100:00000010:2.0:1775764547.960048:0:6160:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at 00000000fb0a686f. 00000100:00000010:2.0:1775764547.960051:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000c10345b7. 00000100:00000010:2.0:1775764547.960053:0:6160:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 00000000bc2b8da8. 00000100:00000010:2.0:1775764547.960055:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000097fb439c. 00000100:00000010:2.0:1775764547.960058:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000f686b69c. 00000100:00000010:2.0:1775764547.960060:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000001559fd7a. 00000100:00000010:2.0:1775764547.960062:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000752c6a14. 00000100:00000010:2.0:1775764547.960064:0:6160:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 00000000fca9d29b. 00000100:00000010:2.0:1775764547.960066:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000072e7fc7e. 00000100:00000010:2.0:1775764547.960068:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000973b40b9. 00000100:00000010:2.0:1775764547.960070:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000b42723bc. 00000100:00000010:2.0:1775764547.960072:0:6160:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at 00000000a14c5615. 00000100:00000010:2.0:1775764547.960074:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000c8f69590. 00000100:00000010:2.0:1775764547.960075:0:6160:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 0000000060097f98. 00000100:00000010:2.0:1775764547.960077:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000016285c32. 00000100:00000010:2.0:1775764547.960079:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000487cfd2c. 00000100:00000010:2.0:1775764547.960083:0:6160:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000002095024. 00000100:00000200:2.0:1775764547.960116:0:6160:0:(service.c:848:ptlrpc_register_service()) ldlm_canceld: Started, listening on portal 17 00000100:00000010:2.0:1775764547.960121:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000a27ea874. 00000400:00000200:0.0:1775764547.961402:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.961411:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : REPLY - for me 00000400:00000200:0.0:1775764547.961442:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.961449:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.961458:0:2393:0:(lib-move.c:4649:lnet_parse_reply()) 192.168.202.4@tcp: Reply from 12345-192.168.202.104@tcp of length 48/48 into md 0x5 00000400:00000200:0.0:1775764547.961478:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764547.961482:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: REPLY: OK 00000400:00000200:0.0:1775764547.961517:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007682e321 00000400:00000010:0.0:1775764547.961521:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000007682e321. 00000400:00000200:0.0:1775764547.961524:0:2393:0:(peer.c:3090:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:0.0:1775764547.961527:0:2393:0:(peer.c:2920:lnet_discovery_event_reply()) Peer 192.168.202.104@tcp has discovery enabled 00000400:00000200:0.0:1775764547.961530:0:2393:0:(peer.c:2932:lnet_discovery_event_reply()) peer 192.168.202.104@tcp(0000000006eda7a3) is MR 00000400:00000200:0.0:1775764547.961533:0:2393:0:(peer.c:3001:lnet_discovery_event_reply()) peer 192.168.202.104@tcp data present 363512030. state = 0x100241 00000400:00000200:0.0:1775764547.961538:0:2393:0:(peer.c:2414:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.104@tcp: -114 00000800:00000010:0.0:1775764547.961684:0:2388:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 0000000029c5bfcf. 00000800:00000010:0.0:1775764547.961689:0:2388:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000007e98606e. 00000800:00000010:0.0:1775764547.961696:0:2388:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000ae762f6a. 00000800:00000010:0.0:1775764547.961706:0:2388:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000ae762f6a. 00000400:00000200:3.0:1775764547.963038:0:2399:0:(peer.c:3997:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:3.0:1775764547.963050:0:2399:0:(peer.c:4109:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x1000c1 00000400:00000010:3.0:1775764547.963063:0:2399:0:(peer.c:3274:lnet_peer_merge_data()) alloc '(curnis)': 40 at 00000000a0e43703. 00000400:00000010:3.0:1775764547.963066:0:2399:0:(peer.c:3275:lnet_peer_merge_data()) alloc '(addnis)': 48 at 0000000099505576. 00000400:00000010:3.0:1775764547.963068:0:2399:0:(peer.c:3276:lnet_peer_merge_data()) alloc '(delnis)': 40 at 0000000052bda3f1. 00000400:00000200:3.0:1775764547.963088:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764547.963094:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764547.963095:0:2399:0:(peer.c:3414:lnet_peer_merge_data()) kfreed 'curnis': 40 at 00000000a0e43703. 00000400:00000010:3.0:1775764547.963098:0:2399:0:(peer.c:3415:lnet_peer_merge_data()) kfreed 'addnis': 48 at 0000000099505576. 00000400:00000010:3.0:1775764547.963100:0:2399:0:(peer.c:3416:lnet_peer_merge_data()) kfreed 'delnis': 40 at 0000000052bda3f1. 00000400:00000010:3.0:1775764547.963103:0:2399:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000ea4f13ad. 00000400:00000200:3.0:1775764547.963107:0:2399:0:(peer.c:3419:lnet_peer_merge_data()) peer 192.168.202.104@tcp (0000000006eda7a3): 0 00000400:00000200:3.0:1775764547.963109:0:2399:0:(peer.c:3723:lnet_peer_data_present()) peer 192.168.202.104@tcp(0000000006eda7a3): 0. state = 0x100141 00000400:00000200:3.0:1775764547.963113:0:2399:0:(peer.c:4127:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100141 rc 1 00000400:00000200:3.0:1775764547.963116:0:2399:0:(peer.c:4109:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100141 00000400:00000010:3.0:1775764547.963122:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000030e77217. 00000400:00000010:3.0:1775764547.963148:0:2399:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at 0000000030ed79aa. 00000400:00000200:3.0:1775764547.963153:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764547.963163:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764547.963169:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source Specified: 192.168.202.4@tcp to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764547.963175:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [2:2:256:4294967295] 192.168.202.104@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:3.0:1775764547.963180:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764547.963184:0:2399:0:(lib-move.c:2011:lnet_handle_send()) rspt_next_hop_nid = 192.168.202.104@tcp 00000400:00000200:3.0:1775764547.963190:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:192.168.202.4@tcp) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764547.963195:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764547.963200:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000070596667. 00000800:00000200:3.0:1775764547.963206:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (3) 00000800:00000200:3.0:1775764547.963212:0:2399:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764547.963217:0:2399:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000070596667 type 1, nob 144 niov 1 nkiov 1 00000800:00000200:0.0:1775764547.964165:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764547.964174:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000070596667. 00000400:00000200:0.0:1775764547.964184:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764547.964193:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764547.964200:0:2393:0:(peer.c:3090:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1775764547.964202:0:2393:0:(peer.c:3042:lnet_discovery_event_send()) Push Send to 192.168.202.104@tcp: 0 00000400:00000200:3.0:1775764547.966243:0:2399:0:(peer.c:3931:lnet_peer_send_push()) peer 192.168.202.104@tcp 00000400:00000200:3.0:1775764547.966249:0:2399:0:(peer.c:4127:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100541 rc 0 00000400:00000200:0.0:1775764547.966596:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:0.1:1775764547.966607:0:2388:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:0.0:1775764547.966615:0:2388:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.202.104@tcp p 3.x 192.168.202.4 -> 192.168.202.104:988 incarnation:1775764356495133599 sched[0] 00000800:00000010:0.0:1775764547.966622:0:2388:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 000000007e98606e. 00000400:00000200:0.0:1775764547.966731:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.202.104:988 00000400:00000200:0.0:1775764547.966802:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.202.104:988 00000100:00000010:1.0:1775764547.967648:0:6165:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000d90719f5. 00000020:00000010:1.0:1775764547.967656:0:6165:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000080f67f63. 00000020:00000010:1.0:1775764547.967677:0:6165:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000013f84e15. 00000020:00000010:1.0:1775764547.967682:0:6165:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008055fc2b. 00000020:00000010:1.0:1775764547.967685:0:6165:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000046de7f59. 00000040:00000010:1.0:1775764547.967690:0:6165:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000069a0452e. 00000020:00000010:1.0:1775764547.967695:0:6165:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000bea89a12. 00000020:00000010:1.0:1775764547.967726:0:6165:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000073489e0. 00000020:00000010:1.0:1775764547.967732:0:6165:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000b37d8f33. 80000000:00000010:1.0:1775764547.967737:0:6165:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000000580160. 40000000:00000010:1.0:1775764547.967740:0:6165:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000d1be6ef3. 00000008:00000010:1.0:1775764547.967759:0:6165:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000078ced8c5. 00020000:00000010:1.0:1775764547.967764:0:6165:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000d26a8c01. 00008000:00000010:1.0:1775764547.967778:0:6165:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e13d1738. 00100000:00000010:1.0:1775764547.967783:0:6165:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000009a290a1a. 00040000:00000010:1.0:1775764547.967788:0:6165:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000094d1a12. 00040000:00000010:1.0:1775764547.967800:0:6165:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000088479ebf. 00040000:00000010:1.0:1775764547.967804:0:6165:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000a8fcffb0. 00000004:00000010:1.0:1775764547.967809:0:6165:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000007c273450. 00000004:00000010:1.0:1775764547.967814:0:6165:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 000000004147f7ce. 00000004:00000010:1.0:1775764547.967819:0:6165:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000fd44f412. 00002000:00000010:1.0:1775764547.967824:0:6165:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000210ef12b. 00000004:00000010:1.0:1775764547.967829:0:6165:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 000000000b966800. 00000080:00000010:1.0:1775764547.967841:0:6165:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000008f3a091a. 00000080:00000010:1.0:1775764547.967856:0:6165:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000774e2b46. 00000100:00000200:1.0:1775764547.967872:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967891:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b88de3f8. 00000400:00000010:1.0:1775764547.967898:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b04c07b3. 00000100:00000200:1.0:1775764547.967907:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967909:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d62bd59e. 00000400:00000010:1.0:1775764547.967912:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003e229d66. 00000100:00000200:1.0:1775764547.967914:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967915:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bec5d296. 00000400:00000010:1.0:1775764547.967918:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009ecd5e49. 00000100:00000200:1.0:1775764547.967920:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967921:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000001c0dd30. 00000400:00000010:1.0:1775764547.967924:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e66da1b4. 00000100:00000200:1.0:1775764547.967926:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967927:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000071ee5d47. 00000400:00000010:1.0:1775764547.967929:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ff2febec. 00000100:00000200:1.0:1775764547.967931:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967932:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f214a011. 00000400:00000010:1.0:1775764547.967934:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006dd9af13. 00000100:00000200:1.0:1775764547.967936:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967938:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000087ed3b74. 00000400:00000010:1.0:1775764547.967946:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006a58b20d. 00000100:00000200:1.0:1775764547.967948:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967950:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000656981dc. 00000400:00000010:1.0:1775764547.967952:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000077234779. 00000100:00000200:1.0:1775764547.967954:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967955:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000141628c4. 00000400:00000010:1.0:1775764547.967957:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000067af4cb. 00000100:00000200:1.0:1775764547.967961:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967962:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003922c1f6. 00000400:00000010:1.0:1775764547.967964:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000066ebb44c. 00000100:00000200:1.0:1775764547.967966:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967968:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000008d3d0b5. 00000400:00000010:1.0:1775764547.967970:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000022ed41f5. 00000100:00000200:1.0:1775764547.967972:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967974:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000060fcf912. 00000400:00000010:1.0:1775764547.967976:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000033161124. 00000100:00000200:1.0:1775764547.967978:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967980:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003acba819. 00000400:00000010:1.0:1775764547.967982:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001cd6a8ef. 00000100:00000200:1.0:1775764547.967984:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967985:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004edd066b. 00000400:00000010:1.0:1775764547.967988:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000032b90873. 00000100:00000200:1.0:1775764547.967990:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967991:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005dbe8acd. 00000400:00000010:1.0:1775764547.967993:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000265035cc. 00000100:00000200:1.0:1775764547.967995:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.967997:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005dc60caf. 00000400:00000010:1.0:1775764547.967999:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005701b152. 00000100:00000200:1.0:1775764547.968001:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968003:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000fe9d626. 00000400:00000010:1.0:1775764547.968005:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004f95ce44. 00000100:00000200:1.0:1775764547.968007:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968008:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009aff65ae. 00000400:00000010:1.0:1775764547.968010:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b2761e2f. 00000100:00000200:1.0:1775764547.968012:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968018:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000014391c7a. 00000400:00000010:1.0:1775764547.968021:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e9c84888. 00000100:00000200:1.0:1775764547.968023:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968026:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f46e4ca1. 00000400:00000010:1.0:1775764547.968029:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e8234097. 00000100:00000200:1.0:1775764547.968031:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968032:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005732e51b. 00000400:00000010:1.0:1775764547.968035:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000089a8e176. 00000100:00000200:1.0:1775764547.968036:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968038:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b4fe9a46. 00000400:00000010:1.0:1775764547.968040:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a3e4e82d. 00000100:00000200:1.0:1775764547.968042:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968044:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ae59d454. 00000400:00000010:1.0:1775764547.968046:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e1b3fc15. 00000100:00000200:1.0:1775764547.968048:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968049:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ad9b186a. 00000400:00000010:1.0:1775764547.968051:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000058ccf99a. 00000100:00000200:1.0:1775764547.968052:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968054:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000066995cf8. 00000400:00000010:1.0:1775764547.968056:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ad7d198f. 00000100:00000200:1.0:1775764547.968058:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968059:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000608864ee. 00000400:00000010:1.0:1775764547.968061:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000027046f0d. 00000100:00000200:1.0:1775764547.968063:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968064:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001a7bafad. 00000400:00000010:1.0:1775764547.968066:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005488cd08. 00000100:00000200:1.0:1775764547.968067:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968069:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003ac701a0. 00000400:00000010:1.0:1775764547.968071:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bcf48dc3. 00000100:00000200:1.0:1775764547.968073:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968075:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c5e76e6f. 00000400:00000010:1.0:1775764547.968078:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d055b40e. 00000100:00000200:1.0:1775764547.968080:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968082:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b0d272c8. 00000400:00000010:1.0:1775764547.968085:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004362ad3a. 00000100:00000200:1.0:1775764547.968092:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968095:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009bcd2eed. 00000400:00000010:1.0:1775764547.968097:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000022b8a19e. 00000100:00000200:1.0:1775764547.968099:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968101:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004ec89c7e. 00000400:00000010:1.0:1775764547.968103:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004d1d91d2. 00000100:00000200:1.0:1775764547.968105:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968106:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000864a6046. 00000400:00000010:1.0:1775764547.968108:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003c584c4c. 00000100:00000200:1.0:1775764547.968110:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968112:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009c7751ef. 00000400:00000010:1.0:1775764547.968114:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001876cfe9. 00000100:00000200:1.0:1775764547.968116:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968118:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003744c49a. 00000400:00000010:1.0:1775764547.968120:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000cd6f1ecd. 00000100:00000200:1.0:1775764547.968122:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968124:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000038a2346f. 00000400:00000010:1.0:1775764547.968126:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000906d180e. 00000100:00000200:1.0:1775764547.968128:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968129:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b32ac801. 00000400:00000010:1.0:1775764547.968131:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e28711fc. 00000100:00000200:1.0:1775764547.968133:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968135:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002ceb1f66. 00000400:00000010:1.0:1775764547.968137:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008342ace9. 00000100:00000200:1.0:1775764547.968139:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968140:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000278bdc89. 00000400:00000010:1.0:1775764547.968142:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000320ecf2c. 00000100:00000200:1.0:1775764547.968144:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968146:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008c10d1e1. 00000400:00000010:1.0:1775764547.968148:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c4f71b7d. 00000100:00000200:1.0:1775764547.968149:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968151:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000058c45c73. 00000400:00000010:1.0:1775764547.968153:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000baa353e. 00000100:00000200:1.0:1775764547.968155:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968156:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000032593501. 00000400:00000010:1.0:1775764547.968165:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006afd6127. 00000100:00000200:1.0:1775764547.968168:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968169:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001a7a058c. 00000400:00000010:1.0:1775764547.968171:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ecdd3132. 00000100:00000200:1.0:1775764547.968173:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968175:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e5a46aeb. 00000400:00000010:1.0:1775764547.968178:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c2c1cc82. 00000100:00000200:1.0:1775764547.968180:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968181:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000a84bcb4. 00000400:00000010:1.0:1775764547.968182:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000068f483db. 00000100:00000200:1.0:1775764547.968184:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968185:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000032fc0c75. 00000400:00000010:1.0:1775764547.968187:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f1fb8100. 00000100:00000200:1.0:1775764547.968188:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968200:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000364fecf6. 00000400:00000010:1.0:1775764547.968204:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ebc34f78. 00000100:00000200:1.0:1775764547.968206:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968208:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a8aaa5b6. 00000400:00000010:1.0:1775764547.968210:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000aa227dbd. 00000100:00000200:1.0:1775764547.968212:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968214:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c036a0cf. 00000400:00000010:1.0:1775764547.968216:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f4445301. 00000100:00000200:1.0:1775764547.968217:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968219:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b2860254. 00000400:00000010:1.0:1775764547.968222:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f5ac0dec. 00000100:00000200:1.0:1775764547.968223:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968225:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f926399c. 00000400:00000010:1.0:1775764547.968227:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000510b5172. 00000100:00000200:1.0:1775764547.968229:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968231:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f06efab3. 00000400:00000010:1.0:1775764547.968233:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009f4f4f13. 00000100:00000200:1.0:1775764547.968235:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968237:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000088c706c2. 00000400:00000010:1.0:1775764547.968239:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e9784ead. 00000100:00000200:1.0:1775764547.968241:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968247:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000308fb9e4. 00000400:00000010:1.0:1775764547.968249:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000113c94bc. 00000100:00000200:1.0:1775764547.968252:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968254:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008a253b35. 00000400:00000010:1.0:1775764547.968255:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005955d2cf. 00000100:00000200:1.0:1775764547.968257:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968259:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000395d6ce8. 00000400:00000010:1.0:1775764547.968261:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000523c76b5. 00000100:00000200:1.0:1775764547.968263:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968265:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000610262e0. 00000400:00000010:1.0:1775764547.968267:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000eb9a5fcf. 00000100:00000200:1.0:1775764547.968268:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968270:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003d5bff79. 00000400:00000010:1.0:1775764547.968272:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003872422f. 00000100:00000200:1.0:1775764547.968274:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968275:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000029ffd70. 00000400:00000010:1.0:1775764547.968277:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001760423f. 00000100:00000200:1.0:1775764547.968279:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968281:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000082637f6e. 00000400:00000010:1.0:1775764547.968283:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ab3282e1. 00000100:00000200:1.0:1775764547.968284:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968286:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000064b3f603. 00000400:00000010:1.0:1775764547.968288:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ac84093f. 00000100:00000200:1.0:1775764547.968290:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968292:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000083373aa2. 00000400:00000010:1.0:1775764547.968293:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007390b694. 00000100:00000200:1.0:1775764547.968295:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968297:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d1587549. 00000400:00000010:1.0:1775764547.968299:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000047fde587. 00000100:00000200:1.0:1775764547.968300:0:6165:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1775764547.968302:0:6165:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008811eda7. 00000400:00000010:1.0:1775764547.968303:0:6165:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a1d230e6. 00000100:00000010:1.0:1775764547.968308:0:6165:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 000000004185cc30. 00000100:00000200:1.0:1775764547.968340:0:6165:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:0.0:1775764547.969211:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000e7f8d7f6. 00000400:00000200:0.0:1775764547.970267:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970275:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : ACK - for me 00000400:00000200:0.0:1775764547.970290:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970295:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970305:0:2394:0:(lib-move.c:4702:lnet_parse_ack()) 192.168.202.4@tcp: ACK from 12345-192.168.202.104@tcp into md 0x15 00000400:00000200:0.0:1775764547.970313:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764547.970317:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: ACK: OK 00000400:00000200:0.0:1775764547.970325:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000030e77217 00000400:00000010:0.0:1775764547.970339:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000030e77217. 00000400:00000200:0.0:1775764547.970345:0:2394:0:(peer.c:3090:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:0.0:1775764547.970348:0:2394:0:(peer.c:2808:lnet_discovery_event_ack()) peer 192.168.202.104@tcp ev->status 0 00000400:00000200:0.0:1775764547.970351:0:2394:0:(peer.c:2414:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.104@tcp: -114 00000400:00000200:0.0:1775764547.970371:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970374:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764547.970378:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970380:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970382:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970385:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 48 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1775764547.970393:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.202.104@tcp of length 48/48 into md 0x9 [1] + 0 00000400:00000010:0.0:1775764547.970397:0:2394:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000009e70d837. 00000400:00000200:0.0:1775764547.970401:0:2394:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000c30a1203 00000400:00000200:0.0:1775764547.970465:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764547.970469:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764547.970472:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c30a1203 00000400:00000010:0.0:1775764547.970474:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c30a1203. 00000400:00000200:0.0:1775764547.970477:0:2394:0:(api-ni.c:2341:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:0.0:1775764547.970481:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970504:0:2394:0:(peer.c:2579:lnet_peer_push_event()) peer 192.168.202.104@tcp(0000000006eda7a3) is MR 00000400:00000010:0.0:1775764547.970509:0:2394:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 0000000047a74c5e. 00000400:00000200:0.0:1775764547.970512:0:2394:0:(peer.c:2653:lnet_peer_push_event()) Received Push 192.168.202.104@tcp 363512030 00000400:00000200:0.0:1775764547.970514:0:2394:0:(peer.c:2414:lnet_peer_queue_for_discovery()) Queue peer 192.168.202.104@tcp: -114 00000400:00000200:0.0:1775764547.970521:0:2394:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:0.0:1775764547.970525:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970529:0:2394:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source Specified: 192.168.202.4@tcp to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764547.970540:0:2394:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [3:3:256:4294967295] 192.168.202.104@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:0.0:1775764547.970545:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764547.970553:0:2394:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:192.168.202.4@tcp) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) 192.168.202.104@tcp : ACK try# 0 00000800:00000200:0.0:1775764547.970559:0:2394:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764547.970563:0:2394:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008034ee17. 00000800:00000200:0.0:1775764547.970570:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (4) 00000800:00000200:0.0:1775764547.970576:0:2394:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764547.970580:0:2394:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000008034ee17 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1775764547.970626:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764547.970629:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008034ee17. 00000400:00000200:0.0:1775764547.970632:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764547.970635:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: ACK: OK 00000400:00000200:3.0:1775764547.971148:0:2399:0:(peer.c:3997:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:3.0:1775764547.971158:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ced8ed9c. 00000400:00000010:3.0:1775764547.971163:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000009a1f06fb. 00000400:00000200:3.0:1775764547.971169:0:2399:0:(api-ni.c:2331:lnet_push_target_post()) posted push target 00000000128152f3 00000400:00000200:3.0:1775764547.971173:0:2399:0:(peer.c:4109:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x1000c1 00000400:00000010:3.0:1775764547.971186:0:2399:0:(peer.c:3274:lnet_peer_merge_data()) alloc '(curnis)': 40 at 0000000099505576. 00000400:00000010:3.0:1775764547.971188:0:2399:0:(peer.c:3275:lnet_peer_merge_data()) alloc '(addnis)': 48 at 00000000a0e43703. 00000400:00000010:3.0:1775764547.971190:0:2399:0:(peer.c:3276:lnet_peer_merge_data()) alloc '(delnis)': 40 at 00000000297dd902. 00000400:00000200:3.0:1775764547.971195:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764547.971199:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764547.971201:0:2399:0:(peer.c:3414:lnet_peer_merge_data()) kfreed 'curnis': 40 at 0000000099505576. 00000400:00000010:3.0:1775764547.971203:0:2399:0:(peer.c:3415:lnet_peer_merge_data()) kfreed 'addnis': 48 at 00000000a0e43703. 00000400:00000010:3.0:1775764547.971205:0:2399:0:(peer.c:3416:lnet_peer_merge_data()) kfreed 'delnis': 40 at 00000000297dd902. 00000400:00000010:3.0:1775764547.971208:0:2399:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 0000000047a74c5e. 00000400:00000200:3.0:1775764547.971211:0:2399:0:(peer.c:3419:lnet_peer_merge_data()) peer 192.168.202.104@tcp (0000000006eda7a3): 0 00000400:00000200:3.0:1775764547.971213:0:2399:0:(peer.c:3723:lnet_peer_data_present()) peer 192.168.202.104@tcp(0000000006eda7a3): 0. state = 0x100141 00000400:00000200:3.0:1775764547.971217:0:2399:0:(peer.c:4127:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100141 rc 1 00000400:00000200:3.0:1775764547.971220:0:2399:0:(peer.c:4109:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100141 00000400:00000200:3.0:1775764547.971222:0:2399:0:(peer.c:3853:lnet_peer_discovered()) peer 192.168.202.104@tcp 00000400:00000200:3.0:1775764547.971224:0:2399:0:(peer.c:4127:lnet_peer_discovery()) peer 192.168.202.104@tcp(0000000006eda7a3) state 0x100111 rc 0 00000400:00000200:3.0:1775764547.971227:0:2399:0:(peer.c:2430:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.202.104@tcp 00000800:00000010:0.0:1775764547.975853:0:2388:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 0000000077b3c2e0. 00000800:00000010:0.0:1775764547.975884:0:2388:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000007e98606e. 00000800:00000010:0.0:1775764547.975896:0:2388:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 0000000065b55831. 00000800:00000010:0.0:1775764547.975908:0:2388:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 0000000065b55831. 00000400:00000200:0.0:1775764547.980095:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:0.1:1775764547.980104:0:2388:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000800:00000200:0.0:1775764547.980115:0:2388:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.202.104@tcp p 3.x 192.168.202.4 -> 192.168.202.104:988 incarnation:1775764356495133599 sched[0] 00000800:00000010:0.0:1775764547.980120:0:2388:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 000000007e98606e. 00000100:00000010:0.0:1775764547.981374:0:6166:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 0000000065b55831. 00000020:00000010:0.0:1775764547.981385:0:6166:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000288e4c93. 00000020:00000010:0.0:1775764547.981395:0:6166:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000002bfc5a52. 00000020:00000010:0.0:1775764547.981407:0:6166:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000008b40d62. 00000020:00000010:0.0:1775764547.981413:0:6166:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000003e459742. 00000040:00000010:0.0:1775764547.981417:0:6166:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000d436f8ba. 00000020:00000010:0.0:1775764547.981422:0:6166:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000058fa60c2. 00000020:00000010:0.0:1775764547.981429:0:6166:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000fa67048a. 00000020:00000010:0.0:1775764547.981433:0:6166:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000fb8fda67. 80000000:00000010:0.0:1775764547.981437:0:6166:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000172024fc. 40000000:00000010:0.0:1775764547.981441:0:6166:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000865acdc8. 00000008:00000010:0.0:1775764547.981480:0:6166:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000087a28c2f. 00020000:00000010:0.0:1775764547.981520:0:6166:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000f1fc241e. 00008000:00000010:0.0:1775764547.981536:0:6166:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000002f1028ef. 00100000:00000010:0.0:1775764547.981541:0:6166:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 0000000025938e34. 00040000:00000010:0.0:1775764547.981546:0:6166:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000008034ee17. 00040000:00000010:0.0:1775764547.981552:0:6166:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000cb3bf079. 00040000:00000010:0.0:1775764547.981556:0:6166:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000018386b86. 00000004:00000010:0.0:1775764547.981561:0:6166:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000023e9c547. 00000004:00000010:0.0:1775764547.981568:0:6166:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000a97f5ff0. 00000004:00000010:0.0:1775764547.981583:0:6166:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000cdf535f1. 00002000:00000010:0.0:1775764547.981588:0:6166:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000df052671. 00000004:00000010:0.0:1775764547.981593:0:6166:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000a5631525. 00000080:00000010:0.0:1775764547.981608:0:6166:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000ae32b6dd. 00000080:00000010:0.0:1775764547.981624:0:6166:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000c93a511c. 00000100:00000010:0.0:1775764547.981630:0:6166:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000e2161e65. 00000100:00000200:0.0:1775764547.981650:0:6166:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1775764547.981804:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000071e5149a. 00000100:00000010:1.0:1775764547.982352:0:6167:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000005ac18329. 00000020:00000010:1.0:1775764547.982362:0:6167:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000007554f0cb. 00000020:00000010:1.0:1775764547.982367:0:6167:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a1781a3f. 00000020:00000010:1.0:1775764547.982370:0:6167:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000000282596f. 00000020:00000010:1.0:1775764547.982373:0:6167:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e71befb3. 00000040:00000010:1.0:1775764547.982376:0:6167:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000069a8ac92. 00000020:00000010:1.0:1775764547.982381:0:6167:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000cf4431c6. 00000020:00000010:1.0:1775764547.982385:0:6167:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000001b3421f7. 00000020:00000010:1.0:1775764547.982389:0:6167:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000002bde4a50. 80000000:00000010:1.0:1775764547.982393:0:6167:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000004661a8e2. 40000000:00000010:1.0:1775764547.982395:0:6167:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000006601f058. 00000008:00000010:1.0:1775764547.982400:0:6167:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000553fa05b. 00020000:00000010:1.0:1775764547.982403:0:6167:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000c70ec8ad. 00008000:00000010:1.0:1775764547.982407:0:6167:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000904faee0. 00100000:00000010:1.0:1775764547.982411:0:6167:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000003d6510f2. 00040000:00000010:1.0:1775764547.982414:0:6167:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000de93bdac. 00040000:00000010:1.0:1775764547.982417:0:6167:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000002ed57097. 00040000:00000010:1.0:1775764547.982419:0:6167:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000071b62d52. 00000004:00000010:1.0:1775764547.982433:0:6167:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000060947df7. 00000004:00000010:1.0:1775764547.982438:0:6167:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000d1b9ffec. 00000004:00000010:1.0:1775764547.982441:0:6167:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000703f5c65. 00002000:00000010:1.0:1775764547.982444:0:6167:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000a9e6ed96. 00000004:00000010:1.0:1775764547.982447:0:6167:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 000000006d51dc26. 00000080:00000010:1.0:1775764547.982450:0:6167:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000f610eec6. 00000080:00000010:1.0:1775764547.982453:0:6167:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d377f600. 00000100:00000010:1.0:1775764547.982457:0:6167:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000ea05f4a9. 00000100:00000200:1.0:1775764547.982476:0:6167:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000010:3.0:1775764547.982553:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000a8d8a316. 00000100:00000010:2.0:1775764547.983179:0:6168:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000008facaa67. 00000020:00000010:2.0:1775764547.983185:0:6168:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000032c70273. 00000020:00000010:2.0:1775764547.983190:0:6168:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000415a5dd4. 00000020:00000010:2.0:1775764547.983193:0:6168:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000006e2d92c8. 00000020:00000010:2.0:1775764547.983196:0:6168:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000ab97ebb4. 00000040:00000010:2.0:1775764547.983200:0:6168:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000008aa6eff4. 00000020:00000010:2.0:1775764547.983203:0:6168:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000e232d885. 00000020:00000010:2.0:1775764547.983222:0:6168:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000007845829f. 00000020:00000010:2.0:1775764547.983227:0:6168:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000ea4f13ad. 80000000:00000010:2.0:1775764547.983231:0:6168:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000075149b9b. 40000000:00000010:2.0:1775764547.983233:0:6168:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000ee980051. 00000008:00000010:2.0:1775764547.983245:0:6168:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000f8da5ca0. 00020000:00000010:2.0:1775764547.983250:0:6168:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000086004c17. 00008000:00000010:2.0:1775764547.983265:0:6168:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000960a208d. 00100000:00000010:2.0:1775764547.983269:0:6168:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000fdd8c527. 00040000:00000010:2.0:1775764547.983272:0:6168:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000008533fd3f. 00040000:00000010:2.0:1775764547.983275:0:6168:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000003d9ccfb3. 00040000:00000010:2.0:1775764547.983280:0:6168:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000000ebd52e2. 00000004:00000010:2.0:1775764547.983283:0:6168:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000a1de12ae. 00000004:00000010:2.0:1775764547.983287:0:6168:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 000000001edae568. 00000004:00000010:2.0:1775764547.983291:0:6168:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000070e69c3. 00002000:00000010:2.0:1775764547.983294:0:6168:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000005b2f47a9. 00000004:00000010:2.0:1775764547.983298:0:6168:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000db7c4fb9. 00000080:00000010:2.0:1775764547.983310:0:6168:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000da365f41. 00000080:00000010:2.0:1775764547.983322:0:6168:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000011fbe667. 00000100:00000200:2.0:1775764547.983446:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983454:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000056e0313c. 00000400:00000010:2.0:1775764547.983461:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008a9b254f. 00000100:00000200:2.0:1775764547.983469:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983471:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000179c3bfb. 00000400:00000010:2.0:1775764547.983474:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004e3d4b0c. 00000100:00000200:2.0:1775764547.983475:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983477:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007a3bd314. 00000400:00000010:2.0:1775764547.983479:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fb209bfe. 00000100:00000200:2.0:1775764547.983481:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983638:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000611d75e1. 00000400:00000010:2.0:1775764547.983643:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002e8688a9. 00000100:00000200:2.0:1775764547.983647:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983650:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000564baaf4. 00000400:00000010:2.0:1775764547.983652:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000019828cf3. 00000100:00000200:2.0:1775764547.983654:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983656:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ac4bc11d. 00000400:00000010:2.0:1775764547.983658:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a163db00. 00000100:00000200:2.0:1775764547.983660:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983671:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000062ab7897. 00000400:00000010:2.0:1775764547.983674:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a920ba27. 00000100:00000200:2.0:1775764547.983676:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983678:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000097584024. 00000400:00000010:2.0:1775764547.983686:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000524d4ec8. 00000100:00000200:2.0:1775764547.983688:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983690:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000000171a15. 00000400:00000010:2.0:1775764547.983692:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005de0cfef. 00000100:00000200:2.0:1775764547.983694:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983695:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001a22f675. 00000400:00000010:2.0:1775764547.983698:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002f0212f8. 00000100:00000200:2.0:1775764547.983700:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983701:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d873b0ec. 00000400:00000010:2.0:1775764547.983703:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b4080127. 00000100:00000200:2.0:1775764547.983706:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983707:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000019f15777. 00000400:00000010:2.0:1775764547.983709:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bca525cc. 00000100:00000200:2.0:1775764547.983711:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983713:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b09129a3. 00000400:00000010:2.0:1775764547.983715:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d9ad9028. 00000100:00000200:2.0:1775764547.983716:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983718:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a7531ef5. 00000400:00000010:2.0:1775764547.983720:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c3adde8f. 00000100:00000200:2.0:1775764547.983723:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983724:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000008be2f1e. 00000400:00000010:2.0:1775764547.983727:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000eb55d212. 00000100:00000200:2.0:1775764547.983729:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983730:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000349f6424. 00000400:00000010:2.0:1775764547.983733:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000000219422. 00000100:00000200:2.0:1775764547.983735:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983736:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000032a9190e. 00000400:00000010:2.0:1775764547.983739:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f8ded940. 00000100:00000200:2.0:1775764547.983741:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983743:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000390e51e1. 00000400:00000010:2.0:1775764547.983745:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000063950678. 00000100:00000200:2.0:1775764547.983747:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983749:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e85f02c7. 00000400:00000010:2.0:1775764547.983751:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008b90e954. 00000100:00000200:2.0:1775764547.983753:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983760:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fb08f563. 00000400:00000010:2.0:1775764547.983763:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000dd7a7172. 00000100:00000200:2.0:1775764547.983766:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983768:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005d126c09. 00000400:00000010:2.0:1775764547.983769:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c0a0090b. 00000100:00000200:2.0:1775764547.983771:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983773:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006b88876a. 00000400:00000010:2.0:1775764547.983775:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000961b214e. 00000100:00000200:2.0:1775764547.983777:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983778:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000daf596c8. 00000400:00000010:2.0:1775764547.983780:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ecac0618. 00000100:00000200:2.0:1775764547.983782:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983785:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000988e7176. 00000400:00000010:2.0:1775764547.983787:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000079655a71. 00000100:00000200:2.0:1775764547.983789:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983790:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000016f1206d. 00000400:00000010:2.0:1775764547.983793:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ed911bf6. 00000100:00000200:2.0:1775764547.983806:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983809:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000085676132. 00000400:00000010:2.0:1775764547.983811:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005cd5b274. 00000100:00000200:2.0:1775764547.983813:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983815:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000abda8129. 00000400:00000010:2.0:1775764547.983817:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d258be89. 00000100:00000200:2.0:1775764547.983819:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983821:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005335443c. 00000400:00000010:2.0:1775764547.983823:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000563b3a03. 00000100:00000200:2.0:1775764547.983825:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983827:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dd658e57. 00000400:00000010:2.0:1775764547.983829:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f03f9bd0. 00000100:00000200:2.0:1775764547.983831:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983832:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004cb6106e. 00000400:00000010:2.0:1775764547.983834:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000002bed95c. 00000100:00000200:2.0:1775764547.983835:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983837:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005915c056. 00000400:00000010:2.0:1775764547.983839:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b1ef949e. 00000100:00000200:2.0:1775764547.983847:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983849:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007a1c9bfa. 00000400:00000010:2.0:1775764547.983852:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000073f55427. 00000100:00000200:2.0:1775764547.983854:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983855:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006caad00d. 00000400:00000010:2.0:1775764547.983857:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000094d9f657. 00000100:00000200:2.0:1775764547.983868:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983870:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a19bec68. 00000400:00000010:2.0:1775764547.983872:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d058e2d9. 00000100:00000200:2.0:1775764547.983874:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983876:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000042726d66. 00000400:00000010:2.0:1775764547.983878:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000093d4ce3a. 00000100:00000200:2.0:1775764547.983880:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983882:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000074c3af79. 00000400:00000010:2.0:1775764547.983884:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000705e90df. 00000100:00000200:2.0:1775764547.983885:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983887:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000335f336a. 00000400:00000010:2.0:1775764547.983889:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007dfb0ea9. 00000100:00000200:2.0:1775764547.983890:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983906:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ed8ea538. 00000400:00000010:2.0:1775764547.983909:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000de95e233. 00000100:00000200:2.0:1775764547.983912:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983914:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c78bfdc5. 00000400:00000010:2.0:1775764547.983916:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000182c9d34. 00000100:00000200:2.0:1775764547.983917:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983919:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000099a9f88a. 00000400:00000010:2.0:1775764547.983921:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d303a37a. 00000100:00000200:2.0:1775764547.983923:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983925:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000071f4e68a. 00000400:00000010:2.0:1775764547.983926:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e2ee14ee. 00000100:00000200:2.0:1775764547.983928:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983930:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000474975d. 00000400:00000010:2.0:1775764547.983931:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000624dcbd. 00000100:00000200:2.0:1775764547.983933:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983935:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003062c4de. 00000400:00000010:2.0:1775764547.983941:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000071703fd1. 00000100:00000200:2.0:1775764547.983943:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983945:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000e9e83ad. 00000400:00000010:2.0:1775764547.983947:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000050a5ad5f. 00000100:00000200:2.0:1775764547.983949:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983951:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ea5dc35a. 00000400:00000010:2.0:1775764547.983952:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a234a74c. 00000100:00000200:2.0:1775764547.983954:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983955:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d0ee82a5. 00000400:00000010:2.0:1775764547.983957:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000031232237. 00000100:00000200:2.0:1775764547.983958:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983960:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000a948290. 00000400:00000010:2.0:1775764547.983962:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004aa0555a. 00000100:00000200:2.0:1775764547.983963:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983965:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba10f5b4. 00000400:00000010:2.0:1775764547.983967:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000082ccd218. 00000100:00000200:2.0:1775764547.983968:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983970:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fd487f85. 00000400:00000010:2.0:1775764547.983972:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000009af9134. 00000100:00000200:2.0:1775764547.983974:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983976:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f19ca685. 00000400:00000010:2.0:1775764547.983978:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000daabc87c. 00000100:00000200:2.0:1775764547.983980:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983982:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001833d72c. 00000400:00000010:2.0:1775764547.983985:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001e105de9. 00000100:00000200:2.0:1775764547.983987:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983989:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c5bcdc35. 00000400:00000010:2.0:1775764547.983991:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000094a0589f. 00000100:00000200:2.0:1775764547.983993:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.983995:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ed14a392. 00000400:00000010:2.0:1775764547.983997:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009efc9927. 00000100:00000200:2.0:1775764547.983999:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984000:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007f0c1864. 00000400:00000010:2.0:1775764547.984002:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c834f4a5. 00000100:00000200:2.0:1775764547.984004:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984012:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000048afed0b. 00000400:00000010:2.0:1775764547.984014:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000beeb7c5a. 00000100:00000200:2.0:1775764547.984017:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984018:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000059588b95. 00000400:00000010:2.0:1775764547.984020:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f2e94179. 00000100:00000200:2.0:1775764547.984023:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984024:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000028761617. 00000400:00000010:2.0:1775764547.984026:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000543e1260. 00000100:00000200:2.0:1775764547.984028:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984030:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4312348. 00000400:00000010:2.0:1775764547.984032:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000045678d09. 00000100:00000200:2.0:1775764547.984034:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984035:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d4b570c1. 00000400:00000010:2.0:1775764547.984037:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000097babbe5. 00000100:00000200:2.0:1775764547.984039:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984040:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000093f4545b. 00000400:00000010:2.0:1775764547.984042:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bb8a7f38. 00000100:00000200:2.0:1775764547.984044:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984045:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4fac131. 00000400:00000010:2.0:1775764547.984048:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000046543efd. 00000100:00000200:2.0:1775764547.984050:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984052:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008e0410c4. 00000400:00000010:2.0:1775764547.984054:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a1476d0f. 00000100:00000200:2.0:1775764547.984056:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984058:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000699e3fb2. 00000400:00000010:2.0:1775764547.984059:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c9ec3262. 00000100:00000200:2.0:1775764547.984061:0:6168:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1775764547.984062:0:6168:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ed66ff73. 00000400:00000010:2.0:1775764547.984064:0:6168:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ee00883f. 00000100:00000010:2.0:1775764547.984068:0:6168:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000053c01ffc. 00000100:00000200:2.0:1775764547.984089:0:6168:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:3.0:1775764547.984343:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000f57cee3c. 00000100:00000010:2.0:1775764547.985269:0:6169:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000009308db4d. 00000020:00000010:2.0:1775764547.985275:0:6169:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000ff97ed1a. 00000020:00000010:2.0:1775764547.985283:0:6169:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e080d201. 00000020:00000010:2.0:1775764547.985287:0:6169:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a6ba07b1. 00000020:00000010:2.0:1775764547.985297:0:6169:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000000f969f40. 00000040:00000010:2.0:1775764547.985301:0:6169:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000000b50f295. 00000020:00000010:2.0:1775764547.985306:0:6169:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000f24bfa24. 00000020:00000010:2.0:1775764547.985310:0:6169:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000006687bf8. 00000020:00000010:2.0:1775764547.985314:0:6169:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000f2b6398d. 80000000:00000010:2.0:1775764547.985318:0:6169:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000f9254ec0. 40000000:00000010:2.0:1775764547.985321:0:6169:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000c24e2153. 00000008:00000010:2.0:1775764547.985327:0:6169:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000d0009407. 00020000:00000010:2.0:1775764547.985331:0:6169:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000007fcd31c1. 00008000:00000010:2.0:1775764547.985336:0:6169:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000191b5d66. 00100000:00000010:2.0:1775764547.985355:0:6169:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000dddeaee3. 00040000:00000010:2.0:1775764547.985361:0:6169:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000d99298c0. 00040000:00000010:2.0:1775764547.985364:0:6169:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000f1944d83. 00040000:00000010:2.0:1775764547.985369:0:6169:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000008e4d2ff5. 00000004:00000010:2.0:1775764547.985373:0:6169:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000c9fb59e0. 00000004:00000010:2.0:1775764547.985379:0:6169:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000a770974a. 00000004:00000010:2.0:1775764547.985384:0:6169:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000061276a85. 00002000:00000010:2.0:1775764547.985388:0:6169:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000669b78b3. 00000004:00000010:2.0:1775764547.985392:0:6169:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000a8693ee7. 00000080:00000010:2.0:1775764547.985397:0:6169:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000007960e1d9. 00000080:00000010:2.0:1775764547.985401:0:6169:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000e616976a. 00000100:00000010:2.0:1775764547.985406:0:6169:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 000000007807d2ce. 00000100:00000200:2.0:1775764547.985430:0:6169:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:3.0:1775764547.985472:0:6160:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000028e52270. 00000100:00000010:3.0:1775764547.986127:0:6170:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000a0e43703. 00000020:00000010:3.0:1775764547.986132:0:6170:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000aaa98fd0. 00000020:00000010:3.0:1775764547.986137:0:6170:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f96f67ce. 00000020:00000010:3.0:1775764547.986145:0:6170:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000063fa832e. 00000020:00000010:3.0:1775764547.986149:0:6170:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000076f1e957. 00000040:00000010:3.0:1775764547.986152:0:6170:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000fcaaa7ed. 00000020:00000010:3.0:1775764547.986157:0:6170:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000b3ae2fa5. 00000020:00000010:3.0:1775764547.986162:0:6170:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000c59394e6. 00000020:00000010:3.0:1775764547.986167:0:6170:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000468efb84. 80000000:00000010:3.0:1775764547.986171:0:6170:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000000dcec920. 40000000:00000010:3.0:1775764547.986173:0:6170:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000002bc7a123. 00000008:00000010:3.0:1775764547.986178:0:6170:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000006db8a89a. 00020000:00000010:3.0:1775764547.986182:0:6170:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000008cf0c8e5. 00008000:00000010:3.0:1775764547.986185:0:6170:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000942138f6. 00100000:00000010:3.0:1775764547.986196:0:6170:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000c5752f17. 00040000:00000010:3.0:1775764547.986199:0:6170:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000b01a4e1a. 00040000:00000010:3.0:1775764547.986213:0:6170:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000dba756e7. 00040000:00000010:3.0:1775764547.986216:0:6170:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000818b41be. 00000004:00000010:3.0:1775764547.986220:0:6170:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000322a86e2. 00000004:00000010:3.0:1775764547.986225:0:6170:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 0000000008dabe24. 00000004:00000010:3.0:1775764547.986229:0:6170:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000ea35bd09. 00002000:00000010:3.0:1775764547.986232:0:6170:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000c422821d. 00000004:00000010:3.0:1775764547.986236:0:6170:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000bb40733a. 00000080:00000010:3.0:1775764547.986239:0:6170:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000006519df48. 00000080:00000010:3.0:1775764547.986243:0:6170:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d8ca9e94. 00000100:00000010:3.0:1775764547.986247:0:6170:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000e198b96a. 00000100:00000200:3.0:1775764547.986259:0:6170:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00010000:00000010:3.0:1775764547.986272:0:6160:0:(ldlm_lockd.c:3466:ldlm_setup()) kmalloced '(blp)': 168 at 00000000cb89afbf. 00010000:00000010:0.0:1775764547.986509:0:6171:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at 00000000f0180764. 00000020:00000010:0.0:1775764547.986514:0:6171:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009033c950. 00000020:00000010:0.0:1775764547.986518:0:6171:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000002246b7b3. 00000020:00000010:0.0:1775764547.986521:0:6171:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000075923bb6. 00000020:00000010:0.0:1775764547.986524:0:6171:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000ebae9397. 00000020:00000010:0.0:1775764547.986528:0:6171:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000fdc49ccb. 80000000:00000010:0.0:1775764547.986531:0:6171:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000085e5ae6f. 40000000:00000010:0.0:1775764547.986533:0:6171:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 0000000007a03767. 00100000:00000010:0.0:1775764547.986536:0:6171:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000564f8402. 00040000:00000010:0.0:1775764547.986539:0:6171:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000003b956118. 00040000:00000010:0.0:1775764547.986541:0:6171:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000003fff14d2. 00002000:00000010:0.0:1775764547.986545:0:6171:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000367c1f74. 00000020:00000010:0.0:1775764547.986548:0:6171:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000c85d074e. 00010000:00000010:2.0:1775764547.986960:0:6172:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at 00000000b9c08a04. 00000020:00000010:2.0:1775764547.986966:0:6172:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e9df9c87. 00000020:00000010:2.0:1775764547.986971:0:6172:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000001b6c763f. 00000020:00000010:2.0:1775764547.986974:0:6172:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000e4d0625a. 00000020:00000010:2.0:1775764547.986978:0:6172:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000d1660e99. 00000020:00000010:2.0:1775764547.986981:0:6172:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000a6011013. 80000000:00000010:2.0:1775764547.986984:0:6172:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000f5aeaa8f. 40000000:00000010:2.0:1775764547.986986:0:6172:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000005afd694f. 00100000:00000010:2.0:1775764547.986989:0:6172:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000004a3ccc21. 00040000:00000010:2.0:1775764547.986992:0:6172:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000001c3c5f38. 00040000:00000010:2.0:1775764547.987001:0:6172:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000055fc4307. 00002000:00000010:2.0:1775764547.987004:0:6172:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000008679bce0. 00000020:00000010:2.0:1775764547.987007:0:6172:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000002d2a44a2. 00000020:00000010:2.0:1775764547.987290:0:6173:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000002e78cb4e. 00000020:00000010:2.0:1775764547.987295:0:6173:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000001de23e39. 00000040:00000010:2.0:1775764547.987298:0:6173:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000ff8ab7ff. 00000020:00000010:2.0:1775764547.987301:0:6173:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000024ad8a12. 00000020:00000010:2.0:1775764547.987304:0:6173:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000004045a0c5. 00000020:00000010:2.0:1775764547.987307:0:6173:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000098e262ed. 80000000:00000010:2.0:1775764547.987309:0:6173:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000002201fb98. 40000000:00000010:2.0:1775764547.987310:0:6173:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000009a88a917. 00100000:00000010:2.0:1775764547.987313:0:6173:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000007e26b3a6. 00040000:00000010:2.0:1775764547.987315:0:6173:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000079ba87c. 00040000:00000010:2.0:1775764547.987318:0:6173:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000f768a836. 00040000:00000010:2.0:1775764547.987320:0:6173:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000687355cc. 00000004:00000010:2.0:1775764547.987323:0:6173:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000867967de. 00000004:00000010:2.0:1775764547.987346:0:6173:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 000000004313deca. 00000004:00000010:2.0:1775764547.987351:0:6173:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000033748f00. 00002000:00000010:2.0:1775764547.987353:0:6173:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000b43c3b8a. 00000004:00000010:2.0:1775764547.987357:0:6173:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 000000005ecf53cb. 00000020:00000010:2.0:1775764547.987360:0:6173:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000cf97912c. 00000001:00000010:3.0:1775764547.987429:0:6160:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000009a54a4d7. 00000001:00000010:3.0:1775764547.987440:0:6160:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000bd4b805d. 00000020:00000010:3.0:1775764547.987471:0:6160:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 000000007fb6cba0. 00000100:00000200:3.0:1775764547.987528:0:6160:0:(events.c:559:ptlrpc_uuid_to_peer()) 192.168.202.104@tcp->12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764547.987538:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764547.987546:0:6160:0:(peer.c:1480:LNetPrimaryNID()) NID 192.168.202.104@tcp primary NID 192.168.202.104@tcp rc 0 00000100:00000010:3.0:1775764547.987553:0:6160:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000d9554ded. 00010000:00000010:3.0:1775764547.987561:0:6160:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000037a2674a. 00010000:00000010:3.0:1775764547.987567:0:6160:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000df4a5e42. 00000001:00000010:3.0:1775764547.987575:0:6160:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000f89f2511. 00000001:00000010:3.0:1775764547.987581:0:6160:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 16 at 00000000f12505ac. 00000001:00000010:3.0:1775764547.987586:0:6160:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at 0000000042f69d7a. 00000001:00000010:3.0:1775764547.987590:0:6160:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at 0000000043f020db. 00000020:00000010:2.0:1775764547.988106:0:6160:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000ba567ae3. 00000020:00000010:2.0:1775764547.988111:0:6160:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 00000000491a1f71. 00000020:00000010:2.0:1775764547.988221:0:6160:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000006bb8a2c8. 00000020:00000010:2.0:1775764547.988224:0:6160:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 000000006d258c1f. 00000040:00000010:2.0:1775764547.988249:0:6160:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 00000000fd9557f5. 00000020:00000010:3.0:1775764548.000475:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000d19d4c82. 00000040:00000010:3.0:1775764548.000513:0:6160:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 0000000095e8b699. 00000020:00000010:3.0:1775764548.000519:0:6160:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 0000000097be601e. 00000020:00000010:3.0:1775764548.000525:0:6160:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 000000001fae4d25. 00040000:00000010:3.0:1775764548.000531:0:6160:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 0000000063523f6d. 00040000:00000010:3.0:1775764548.000535:0:6160:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000b5d79eeb. 00000020:00000010:3.0:1775764548.000548:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c690e0af. 00000020:00000010:3.0:1775764548.000555:0:6160:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000001230894f. 00000020:00000010:3.0:1775764548.000559:0:6160:0:(tgt_main.c:769:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000067181d6f. 00000008:00000010:3.0:1775764548.000564:0:6160:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000a021b43c. 00020000:00000010:3.0:1775764548.000569:0:6160:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000003da9f5d9. 00008000:00000010:3.0:1775764548.000574:0:6160:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000004d2cc17. 00000080:00000010:3.0:1775764548.000578:0:6160:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000b5e74327. 00000020:00000010:3.0:1775764548.000581:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e07977ad. 00000020:00000010:3.0:1775764548.000586:0:6160:0:(obd_mount.c:248:lustre_start_simple()) kfreed 'lcfg': 104 at 00000000d0b619fb. 00000020:00000010:3.0:1775764548.000604:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000003c3a54d0. 00000400:00000200:3.0:1775764548.000630:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.000640:0:6160:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.104@tcp is discovered: rc = 1 00000100:00000010:3.0:1775764548.000683:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005d25921b. 02000000:00000010:3.0:1775764548.000702:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000783f7f3e. 00000020:00000010:3.0:1775764548.000741:0:6160:0:(obd_mount.c:537:lustre_start_mgc()) kfreed 'uuid': 40 at 0000000090cbab9b. 00000020:00000010:3.0:1775764548.000745:0:6160:0:(obd_mount.c:538:lustre_start_mgc()) kfreed 'data': 192 at 00000000022c6efc. 00000020:00000010:3.0:1775764548.000748:0:6160:0:(obd_mount.c:539:lustre_start_mgc()) kfreed 'mgcname': 23 at 0000000079c95200. 00000080:00000010:3.0:1775764548.000774:0:6160:0:(llite_lib.c:1370:ll_fill_super()) kmalloced '(cfg)': 88 at 00000000822edfe2. 00000080:00000010:3.0:1775764548.000779:0:6160:0:(llite_lib.c:93:ll_init_sbi()) kmalloced '(sbi)': 1376 at 000000001050d395. 00000020:00000010:3.0:1775764548.000877:0:6160:0:(cl_page.c:1297:cl_cache_init()) kmalloced '(cache)': 152 at 0000000049df0dc5. 00000080:00000010:3.0:1775764548.000881:0:6160:0:(llite_lib.c:127:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_prefix)': 6 at 0000000060ef66d5. 00000080:00000010:3.0:1775764548.000884:0:6160:0:(llite_lib.c:134:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_upcall)': 5 at 000000004f6d5966. 00000020:00000010:3.0:1775764548.001320:0:6160:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000726f2922. 00000020:00000010:3.0:1775764548.001326:0:6160:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 1440 at 00000000d30d7f88. 00000020:00000010:3.0:1775764548.001357:0:6160:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000a03be26f. 00000020:00000010:3.0:1775764548.001360:0:6160:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 576 at 0000000067ec4e96. 00000020:00000010:3.0:1775764548.001370:0:6160:0:(obd_mount.c:66:lustre_process_log()) kmalloced '(bufs)': 104 at 00000000d0b619fb. 00000020:00000010:3.0:1775764548.001374:0:6160:0:(obd_mount.c:75:lustre_process_log()) kmalloced '(lcfg)': 184 at 00000000241731e7. 00000020:00000010:3.0:1775764548.001378:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c690e0af. 00000020:00000010:3.0:1775764548.001381:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000001fae4d25. 00000040:00000010:3.0:1775764548.001385:0:6160:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000095e8b699. 00000020:00000010:3.0:1775764548.001388:0:6160:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000000269a532. 00000020:00000010:3.0:1775764548.001399:0:6160:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000d19d4c82. 00040000:00000010:3.0:1775764548.001403:0:6160:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000058a9d46. 00040000:00000010:3.0:1775764548.001407:0:6160:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000955b184b. 10000000:00000010:3.0:1775764548.001419:0:6160:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 287 at 0000000038ace36e. 02000000:00000010:3.0:1775764548.001427:0:6160:0:(sec_config.c:546:sptlrpc_conf_get()) kmalloced '(conf)': 120 at 000000001230894f. 00000100:00000010:3.0:1775764548.001433:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004d392ffa. 02000000:00000010:3.0:1775764548.001441:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 0000000009ba2065. 00010000:00000010:3.0:1775764548.001453:0:6160:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000304b2843. 00010000:00000010:3.0:1775764548.001459:0:6160:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000eaccb8d5. 00000020:00000010:3.0:1775764548.001464:0:6160:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at 0000000054f8842e. 00000100:00000010:3.0:1775764548.001511:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000004edc4c48. 00000020:00000010:0.0:1775764548.004399:0:2400:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000001a6cde91. 00000020:00000010:0.0:1775764548.004418:0:2400:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000060c9bb41. 80000000:00000010:0.0:1775764548.004423:0:2400:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000001eb9af2. 40000000:00000010:0.0:1775764548.004428:0:2400:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000000c63a29c. 00000008:00000010:0.0:1775764548.004443:0:2400:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000002e29bc05. 00020000:00000010:0.0:1775764548.004449:0:2400:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000e5332958. 00008000:00000010:0.0:1775764548.004455:0:2400:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000640254a7. 00100000:00000010:0.0:1775764548.004460:0:2400:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000044d9d6e. 00040000:00000010:0.0:1775764548.004466:0:2400:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000dff35fbb. 00040000:00000010:0.0:1775764548.004471:0:2400:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000091e52ed1. 00040000:00000010:0.0:1775764548.004474:0:2400:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000008c76171d. 00000004:00000010:0.0:1775764548.004479:0:2400:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000008b722c9c. 00000004:00000010:0.0:1775764548.004504:0:2400:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000d1664db1. 00000004:00000010:0.0:1775764548.004510:0:2400:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000008d837e4c. 00002000:00000010:0.0:1775764548.004515:0:2400:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000033827b98. 00000004:00000010:0.0:1775764548.004519:0:2400:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000aeda97d3. 00000080:00000010:0.0:1775764548.004524:0:2400:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000019123e96. 00000080:00000010:0.0:1775764548.004530:0:2400:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000309575a2. 00000008:00000010:0.0:1775764548.004552:0:2400:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 0000000060897433. 00020000:00000010:0.0:1775764548.004567:0:2400:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000008939987c. 00008000:00000010:0.0:1775764548.004588:0:2400:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000015b5f4b3. 00000080:00000010:0.0:1775764548.004599:0:2400:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 0000000090fbc2e7. 02000000:00000010:0.0:1775764548.004625:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000b840905e. 00000400:00000200:0.0:1775764548.004637:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:0.0:1775764548.004641:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000a6ece6c. 00000400:00000010:0.0:1775764548.004658:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cccb5ed9. 00000100:00000200:0.0:1775764548.004681:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861239936, portal 25 00000400:00000010:0.0:1775764548.004689:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000911c629a. 00000100:00000200:0.0:1775764548.004692:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862023861239936, offset 0 00000400:00000200:0.0:1775764548.004704:0:2400:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:0.0:1775764548.004716:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.004725:0:2400:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764548.004730:0:2400:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775764548.004734:0:2400:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:0.0:1775764548.004739:0:2400:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775764548.004741:0:2400:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:0.0:1775764548.004745:0:2400:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [4:4:256:4294967295] 192.168.202.104@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:0.0:1775764548.004755:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.004765:0:2400:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:0.0:1775764548.004771:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764548.004775:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000079df77b0. 00000800:00000200:0.0:1775764548.004782:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:0.0:1775764548.004791:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764548.005079:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000079df77b0 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1775764548.005099:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:0.0:1775764548.005107:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000f1a43e15. 00000100:00000010:0.0:1775764548.005127:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000102c4d2e. 00000800:00000200:0.0:1775764548.005294:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.005298:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000079df77b0. 00000400:00000200:0.0:1775764548.005303:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.005309:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.005315:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000911c629a 00000400:00000010:0.0:1775764548.005317:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000911c629a. 00000100:00000200:0.0:1775764548.005327:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005d25921b x1862023861239936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 1775764558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775764548.008552:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000f1a43e15. 00000100:00000010:1.0:1775764548.012035:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000102c4d2e. 00000400:00000200:0.0:1775764548.013636:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.013647:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.013656:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.013759:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.013769:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.013774:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 416 into portal 25 MB=0x69d8036d00080 00000400:00000200:0.0:1775764548.013780:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.013787:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 416/416 into md 0x221 [1] + 0 00000400:00000200:0.0:1775764548.013905:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.013912:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.013928:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005d25921b x1862023861239936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775764548.014001:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000000a6ece6c. 00000400:00000200:0.0:1775764548.014006:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cccb5ed9 00000400:00000010:0.0:1775764548.014009:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000cccb5ed9. 00000100:00000200:0.0:1775764548.014014:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005d25921b x1862023861239936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775764548.014023:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005d25921b x1862023861239936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775764548.014833:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000b840905e. 02000000:00000010:0.0:1775764548.014845:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000783f7f3e. 00000100:00000010:0.0:1775764548.014853:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005d25921b. 02000000:00000010:2.0:1775764548.020275:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f29b6b28. 00000400:00000200:2.0:1775764548.020303:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764548.020308:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dc311c9f. 00000400:00000010:2.0:1775764548.020317:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000edaafa3c. 00000100:00000200:2.0:1775764548.020326:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861240064, portal 25 00000400:00000010:2.0:1775764548.020333:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c64b9cac. 00000100:00000200:2.0:1775764548.020338:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1862023861240064, offset 0 00000400:00000200:2.0:1775764548.020360:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.020374:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.020384:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764548.020391:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764548.020398:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764548.020404:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764548.020408:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764548.020412:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [5:5:256:4294967295] 192.168.202.104@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:2.0:1775764548.020418:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.020430:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764548.020436:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764548.020442:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b54b7ea0. 00000800:00000200:2.0:1775764548.020452:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764548.020462:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764548.020469:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000b54b7ea0 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:2.0:1775764548.020521:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:2.0:1775764548.020529:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000eab241e8. 00000100:00000010:2.0:1775764548.020550:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000046048fe6. 00000800:00000200:0.0:1775764548.024061:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.024069:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b54b7ea0. 00000400:00000200:0.0:1775764548.024079:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.024091:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.024100:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c64b9cac 00000400:00000010:0.0:1775764548.024104:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c64b9cac. 00000100:00000200:0.0:1775764548.024116:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004d392ffa x1862023861240064/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775764548.024250:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000eab241e8. 00000100:00000010:1.0:1775764548.026401:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000046048fe6. 00000400:00000200:0.0:1775764548.027915:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.027923:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.027933:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.027939:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.027944:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.027948:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 344 into portal 25 MB=0x69d8036d00100 00000400:00000200:0.0:1775764548.027954:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.027959:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 344/344 into md 0x231 [1] + 224 00000400:00000200:0.0:1775764548.027974:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.027979:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.027992:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004d392ffa x1862023861240064/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775764548.028113:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000dc311c9f. 00000400:00000200:3.0:1775764548.028119:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000edaafa3c 00000400:00000010:3.0:1775764548.028121:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000edaafa3c. 00000100:00000200:3.0:1775764548.028126:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004d392ffa x1862023861240064/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775764548.028135:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004d392ffa x1862023861240064/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775764548.028164:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000004edc4c48. 02000000:00000010:3.0:1775764548.028196:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f29b6b28. 02000000:00000010:3.0:1775764548.028202:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 0000000009ba2065. 00000100:00000010:3.0:1775764548.028208:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004d392ffa. 10000000:00000010:3.0:1775764548.028218:0:6160:0:(mgc_request.c:1700:mgc_process_cfg_log()) kmalloced '(env)': 56 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.028224:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000009ba2065. 00000020:00000010:3.0:1775764548.028230:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b5d79eeb. 00000040:00000010:3.0:1775764548.028233:0:6160:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000e7c7201f. 00000020:00000010:3.0:1775764548.028239:0:6160:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000005b999265. 00000020:00000010:3.0:1775764548.028243:0:6160:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000777d7d11. 80000000:00000010:3.0:1775764548.028247:0:6160:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000097e1f0f6. 00040000:00000010:3.0:1775764548.028251:0:6160:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000580903b4. 20000000:00000010:3.0:1775764548.028255:0:6160:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 00000000df020cbc. 00000040:00000010:3.0:1775764548.028266:0:6160:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 000000004636cd07. 00000100:00000010:3.0:1775764548.028278:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004d392ffa. 02000000:00000010:3.0:1775764548.028294:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000d8c0e457. 00000100:00000010:3.0:1775764548.028308:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000009e6a7685. 02000000:00000010:3.0:1775764548.028327:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000147c0326. 00000400:00000200:3.0:1775764548.028332:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764548.028334:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003cbaac95. 00000400:00000010:3.0:1775764548.028339:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007371dadb. 00000100:00000200:3.0:1775764548.028345:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1862023861240192, portal 25 00000400:00000010:3.0:1775764548.028351:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000048c2cf13. 00000100:00000200:3.0:1775764548.028354:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1862023861240192, offset 0 00000400:00000200:3.0:1775764548.028359:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764548.028365:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.028372:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764548.028379:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775764548.028383:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:3.0:1775764548.028387:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775764548.028390:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.028393:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [6:6:256:4294967295] 192.168.202.104@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:3.0:1775764548.028397:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.028408:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764548.028414:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764548.028419:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006e5f5541. 00000800:00000200:3.0:1775764548.028425:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:3.0:1775764548.028432:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764548.028436:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000006e5f5541 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:3.0:1775764548.028454:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:3.0:1775764548.028459:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000074dd89e4. 00000100:00000010:3.0:1775764548.028471:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000b7f4c198. 00000800:00000200:0.0:1775764548.029921:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.029928:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006e5f5541. 00000400:00000200:0.0:1775764548.029935:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.029943:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.029948:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000048c2cf13 00000400:00000010:0.0:1775764548.029950:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000048c2cf13. 00000100:00000200:0.0:1775764548.029958:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004d392ffa x1862023861240192/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775764548.031804:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000074dd89e4. 00000100:00000010:1.0:1775764548.032261:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000b7f4c198. 00000400:00000200:0.0:1775764548.039318:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.039329:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.039339:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.039344:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.039351:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.039355:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 272 into portal 25 MB=0x69d8036d00180 00000400:00000200:0.0:1775764548.039362:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.039370:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 272/272 into md 0x239 [1] + 224 00000400:00000200:0.0:1775764548.039391:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.039396:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.039412:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004d392ffa x1862023861240192/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775764548.039855:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000003cbaac95. 00000400:00000200:3.0:1775764548.039869:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007371dadb 00000400:00000010:3.0:1775764548.039872:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000007371dadb. 00000100:00000200:3.0:1775764548.039893:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004d392ffa x1862023861240192/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775764548.039903:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004d392ffa x1862023861240192/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775764548.039933:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000009e6a7685. 02000000:00000010:3.0:1775764548.039944:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000147c0326. 02000000:00000010:3.0:1775764548.039949:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000d8c0e457. 00000100:00000010:3.0:1775764548.039953:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004d392ffa. 00000040:00000010:3.0:1775764548.039958:0:6160:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at 000000004636cd07. 00000020:00000010:3.0:1775764548.039968:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000b5d79eeb. 00000040:00000010:3.0:1775764548.039973:0:6160:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000e7c7201f. 00000020:00000010:3.0:1775764548.039977:0:6160:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 000000005b999265. 00000020:00000010:3.0:1775764548.039983:0:6160:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000777d7d11. 80000000:00000010:3.0:1775764548.039988:0:6160:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 0000000097e1f0f6. 00040000:00000010:3.0:1775764548.039993:0:6160:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000580903b4. 20000000:00000010:3.0:1775764548.039999:0:6160:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at 00000000df020cbc. 00000020:00000010:3.0:1775764548.040002:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000009ba2065. 10000000:00000010:3.0:1775764548.040006:0:6160:0:(mgc_request.c:1748:mgc_process_cfg_log()) kfreed 'env': 56 at 00000000e51b4ac5. 10000000:00000010:3.0:1775764548.040037:0:6160:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 279 at 00000000d8c0e457. 10000000:00000010:3.0:1775764548.040043:0:6160:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 286 at 00000000147c0326. 10000000:00000010:3.0:1775764548.040047:0:6160:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 285 at 000000004e10fc76. 00000100:00000010:3.0:1775764548.040053:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004d392ffa. 02000000:00000010:3.0:1775764548.040066:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000009e8cfe63. 00010000:00000010:3.0:1775764548.040085:0:6160:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000a2448733. 00000100:00000010:3.0:1775764548.040106:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000009e6a7685. 02000000:00000010:3.0:1775764548.040126:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000034e2787b. 00000400:00000200:3.0:1775764548.040132:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764548.040135:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003cbaac95. 00000400:00000010:3.0:1775764548.040139:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007371dadb. 00000100:00000200:3.0:1775764548.040144:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861240320, portal 25 00000400:00000010:3.0:1775764548.040149:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d3525c8a. 00000100:00000200:3.0:1775764548.040153:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1862023861240320, offset 0 00000400:00000200:3.0:1775764548.040166:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764548.040172:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.040177:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764548.040182:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775764548.040186:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:3.0:1775764548.040190:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775764548.040193:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.040196:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [7:7:256:4294967295] 192.168.202.104@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:3.0:1775764548.040200:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.040208:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764548.040214:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764548.040219:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000e15767c0. 00000800:00000200:3.0:1775764548.040226:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:3.0:1775764548.040232:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764548.040236:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000e15767c0 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1775764548.040246:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:3.0:1775764548.040251:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000097e1f0f6. 00000100:00000010:3.0:1775764548.040264:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a68b0f9d. 00000800:00000200:0.0:1775764548.040540:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.040544:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000e15767c0. 00000400:00000200:0.0:1775764548.040548:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.040553:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.040556:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d3525c8a 00000400:00000010:0.0:1775764548.040558:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000d3525c8a. 00000100:00000200:0.0:1775764548.040563:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000004d392ffa x1862023861240320/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.043578:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a68b0f9d. 00000100:00000010:0.0:1775764548.044075:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000097e1f0f6. 00000400:00000200:0.0:1775764548.052914:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.052923:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.052933:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.052939:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.052944:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.052948:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 344 into portal 25 MB=0x69d8036d00200 00000400:00000200:0.0:1775764548.052954:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.052960:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 344/344 into md 0x241 [1] + 224 00000400:00000200:0.0:1775764548.052984:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.052989:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.053006:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000004d392ffa x1862023861240320/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775764548.055694:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000003cbaac95. 00000400:00000200:2.0:1775764548.055709:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007371dadb 00000400:00000010:2.0:1775764548.055711:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000007371dadb. 00000100:00000200:2.0:1775764548.055729:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000004d392ffa x1862023861240320/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.055741:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000004d392ffa x1862023861240320/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764548.055778:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000009e6a7685. 02000000:00000010:2.0:1775764548.055820:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000034e2787b. 02000000:00000010:2.0:1775764548.055827:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000009e8cfe63. 00000100:00000010:2.0:1775764548.055832:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000004d392ffa. 10000000:00000010:2.0:1775764548.055848:0:6160:0:(mgc_request.c:1700:mgc_process_cfg_log()) kmalloced '(env)': 56 at 000000000e92995b. 00000020:00000010:2.0:1775764548.055871:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000a45a8e4. 00000020:00000010:2.0:1775764548.055880:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000dc578cdd. 00000040:00000010:2.0:1775764548.055886:0:6160:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000001e426763. 00000020:00000010:2.0:1775764548.055891:0:6160:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000f602d917. 00000020:00000010:2.0:1775764548.055913:0:6160:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000c7915c02. 80000000:00000010:2.0:1775764548.055918:0:6160:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000002a4f10c1. 00040000:00000010:2.0:1775764548.055924:0:6160:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000a95ec3e3. 20000000:00000010:2.0:1775764548.055929:0:6160:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 0000000031e063b1. 00000040:00000010:2.0:1775764548.055937:0:6160:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 00000000a3f385fa. 00000100:00000010:2.0:1775764548.055957:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000277fa9c1. 02000000:00000010:2.0:1775764548.055976:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000009f8d13b4. 00000100:00000010:2.0:1775764548.055990:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000090526115. 02000000:00000010:2.0:1775764548.056011:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000dcc71704. 00000400:00000200:2.0:1775764548.056021:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764548.056025:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006f22505c. 00000400:00000010:2.0:1775764548.056030:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000df870992. 00000100:00000200:2.0:1775764548.056037:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1862023861240448, portal 25 00000400:00000010:2.0:1775764548.056041:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ae35d418. 00000100:00000200:2.0:1775764548.056043:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1862023861240448, offset 0 00000400:00000200:2.0:1775764548.056049:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.056058:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.056066:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764548.056071:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764548.056075:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764548.056080:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764548.056083:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764548.056087:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [8:8:256:4294967295] 192.168.202.104@tcp LPNI seq info [8:8:8:4294967295] 00000400:00000200:2.0:1775764548.056090:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.056100:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764548.056106:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764548.056113:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009d239544. 00000800:00000200:2.0:1775764548.056121:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764548.056134:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764548.056139:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000009d239544 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:2.0:1775764548.056159:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:2.0:1775764548.056164:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000ef630302. 00000100:00000010:2.0:1775764548.056182:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000003629c727. 00000800:00000200:0.0:1775764548.057017:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.057024:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009d239544. 00000400:00000200:0.0:1775764548.057032:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.057041:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.057047:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ae35d418 00000400:00000010:0.0:1775764548.057050:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000ae35d418. 00000100:00000200:0.0:1775764548.057058:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000277fa9c1 x1862023861240448/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.060036:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000003629c727. 00000400:00000200:1.0:1775764548.060085:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.060093:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.060140:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.060145:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.060152:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.060157:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 272 into portal 25 MB=0x69d8036d00280 00000400:00000200:1.0:1775764548.060162:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.060169:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 272/272 into md 0x249 [1] + 224 00000400:00000200:1.0:1775764548.060191:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.060196:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.060210:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000277fa9c1 x1862023861240448/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775764548.060329:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000006f22505c. 00000400:00000200:2.0:1775764548.060335:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000df870992 00000400:00000010:2.0:1775764548.060337:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000df870992. 00000100:00000200:2.0:1775764548.060344:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000277fa9c1 x1862023861240448/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.060352:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000277fa9c1 x1862023861240448/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 512/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764548.060388:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000090526115. 02000000:00000010:2.0:1775764548.060404:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000dcc71704. 02000000:00000010:2.0:1775764548.060409:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000009f8d13b4. 00000100:00000010:2.0:1775764548.060414:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000277fa9c1. 00000040:00000010:2.0:1775764548.060428:0:6160:0:(llog.c:415:llog_init_handle()) kmalloced '(llh)': 8192 at 00000000702bfb51. 00000100:00000010:2.0:1775764548.060434:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000277fa9c1. 02000000:00000010:2.0:1775764548.060448:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000009f8d13b4. 00000100:00000010:2.0:1775764548.060460:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000090526115. 02000000:00000010:2.0:1775764548.060531:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 00000000424fd1fd. 00000400:00000200:2.0:1775764548.060537:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764548.060540:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006f22505c. 00000400:00000010:2.0:1775764548.060545:0:6160:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 00000000b684c81e. 00000100:00000200:2.0:1775764548.060552:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1862023861240576, portal 25 00000400:00000010:2.0:1775764548.060557:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000df870992. 00000100:00000200:2.0:1775764548.060560:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1862023861240576, offset 0 00000400:00000200:2.0:1775764548.060568:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.060574:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.060580:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764548.060585:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764548.060596:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764548.060600:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764548.060604:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764548.060609:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [9:9:256:4294967295] 192.168.202.104@tcp LPNI seq info [9:9:8:4294967295] 00000400:00000200:2.0:1775764548.060614:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.060623:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764548.060630:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764548.060636:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcf3a1f9. 00000800:00000200:2.0:1775764548.060643:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764548.060652:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764548.060657:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000dcf3a1f9 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1775764548.060685:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:2.0:1775764548.060699:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005fb9b737. 00000800:00000200:0.0:1775764548.063028:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.063036:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcf3a1f9. 00000400:00000200:0.0:1775764548.063043:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.063053:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.063061:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000df870992 00000400:00000010:0.0:1775764548.063063:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000df870992. 00000100:00000200:0.0:1775764548.063077:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000277fa9c1 x1862023861240576/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.065317:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005fb9b737. 00000100:00000010:0.0:1775764548.065425:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000ef630302. 00000400:00000200:0.0:1775764548.072272:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.072282:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.072310:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.072315:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.072321:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.072325:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 8416 into portal 25 MB=0x69d8036d00300 00000400:00000200:0.0:1775764548.072330:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.072337:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 8416/8416 into md 0x251 [4] + 224 00000400:00000200:0.0:1775764548.072360:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.072365:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.072380:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000277fa9c1 x1862023861240576/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775764548.072645:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000006f22505c. 00000400:00000200:2.0:1775764548.072650:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b684c81e 00000400:00000010:2.0:1775764548.072653:0:6160:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 00000000b684c81e. 00000100:00000200:2.0:1775764548.072659:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000277fa9c1 x1862023861240576/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.072765:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000277fa9c1 x1862023861240576/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764548.072881:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000090526115. 02000000:00000010:2.0:1775764548.072896:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 00000000424fd1fd. 02000000:00000010:2.0:1775764548.072913:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000009f8d13b4. 00000100:00000010:2.0:1775764548.072918:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000277fa9c1. 00000040:00000010:2.0:1775764548.072934:0:6160:0:(llog.c:948:llog_process_or_fork()) kmalloced '(lpi)': 96 at 000000004f52e8d5. 00000020:00000010:0.0:1775764548.073200:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000002884ac61. 00000020:00000010:0.0:1775764548.073206:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008947adc5. 00000040:00000010:0.0:1775764548.073210:0:6179:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000009bbda9a2. 00000020:00000010:0.0:1775764548.073214:0:6179:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000051531c9. 00000020:00000010:0.0:1775764548.073220:0:6179:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000acc5946a. 00000020:00000010:0.0:1775764548.073242:0:6179:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000053d89429. 80000000:00000010:0.0:1775764548.073248:0:6179:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000f1a43e15. 40000000:00000010:0.0:1775764548.073252:0:6179:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 0000000047a74c5e. 00100000:00000010:0.0:1775764548.073258:0:6179:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000009edb78ef. 00040000:00000010:0.0:1775764548.073264:0:6179:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000017a02ccd. 00040000:00000010:0.0:1775764548.073268:0:6179:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000eed95878. 00040000:00000010:0.0:1775764548.073271:0:6179:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000061f83f4b. 00000004:00000010:0.0:1775764548.073276:0:6179:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000044f57bb. 00000004:00000010:0.0:1775764548.073282:0:6179:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000b6fdc258. 00000004:00000010:0.0:1775764548.073286:0:6179:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000004de738fa. 00002000:00000010:0.0:1775764548.073291:0:6179:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000007ca6751b. 00000004:00000010:0.0:1775764548.073295:0:6179:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000bc041687. 00000020:00000010:0.0:1775764548.073301:0:6179:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000007582954d. 00000040:00000010:0.0:1775764548.073309:0:6179:0:(llog.c:573:llog_process_thread()) kmalloced '(buf)': 8192 at 000000000d38ef68. 00000100:00000010:0.0:1775764548.073327:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007245d88c. 02000000:00000010:0.0:1775764548.073341:0:6179:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000004689d2f8. 00000100:00000010:0.0:1775764548.073359:0:6179:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000006359a315. 02000000:00000010:0.0:1775764548.073388:0:6179:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 00000000a9f43a5d. 00000400:00000200:0.0:1775764548.073395:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:0.0:1775764548.073400:0:6179:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000a6ece6c. 00000400:00000010:0.0:1775764548.073405:0:6179:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 0000000093467089. 00000100:00000200:0.0:1775764548.073413:0:6179:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1862023861240704, portal 25 00000400:00000010:0.0:1775764548.073420:0:6179:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cccb5ed9. 00000100:00000200:0.0:1775764548.073422:0:6179:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1862023861240704, offset 0 00000400:00000200:0.0:1775764548.073430:0:6179:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:0.0:1775764548.073437:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.073444:0:6179:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764548.073449:0:6179:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775764548.073453:0:6179:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:0.0:1775764548.073458:0:6179:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775764548.073461:0:6179:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:0.0:1775764548.073465:0:6179:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [10:10:256:4294967295] 192.168.202.104@tcp LPNI seq info [10:10:8:4294967295] 00000400:00000200:0.0:1775764548.073469:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.073478:0:6179:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:0.0:1775764548.073603:0:6179:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764548.073609:0:6179:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b3ab7032. 00000800:00000200:0.0:1775764548.073616:0:6179:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:0.0:1775764548.073625:0:6179:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764548.073630:0:6179:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000b3ab7032 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:0.0:1775764548.073783:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:0.0:1775764548.073789:0:6179:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002459c4af. 00000100:00000010:0.0:1775764548.073889:0:6179:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000005e7c03b3. 00000800:00000200:1.0:1775764548.074761:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.074768:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b3ab7032. 00000400:00000200:1.0:1775764548.074778:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.074786:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.074794:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cccb5ed9 00000400:00000010:1.0:1775764548.074797:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000cccb5ed9. 00000100:00000200:1.0:1775764548.074811:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000007245d88c x1862023861240704/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.077782:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000005e7c03b3. 00000400:00000200:0.0:1775764548.079096:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.079107:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.079117:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.079122:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.079129:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.079153:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 8472 into portal 25 MB=0x69d8036d00380 00000400:00000200:0.0:1775764548.079159:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.079165:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 8472/8472 into md 0x259 [4] + 224 00000400:00000200:0.0:1775764548.079185:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.079188:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.079199:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000007245d88c x1862023861240704/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775764548.079274:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002459c4af. 00000400:00000010:3.0:1775764548.079308:0:6179:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000000a6ece6c. 00000400:00000200:3.0:1775764548.079313:0:6179:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000093467089 00000400:00000010:3.0:1775764548.079316:0:6179:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 0000000093467089. 00000100:00000200:3.0:1775764548.079321:0:6179:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000007245d88c x1862023861240704/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775764548.079331:0:6179:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000007245d88c x1862023861240704/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775764548.079364:0:6179:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000006359a315. 02000000:00000010:3.0:1775764548.079376:0:6179:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 00000000a9f43a5d. 02000000:00000010:3.0:1775764548.079389:0:6179:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000004689d2f8. 00000100:00000010:3.0:1775764548.079394:0:6179:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000007245d88c. 00000020:00000010:3.0:1775764548.079412:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000c9a834a7. 00000020:00000010:3.0:1775764548.079421:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000c9a834a7. 00000020:00000010:3.0:1775764548.079425:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.079429:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.079440:0:6179:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6592 at 000000003c6539b2. 00000020:00000010:3.0:1775764548.079447:0:6179:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000005b999265. 00000020:00000010:3.0:1775764548.079465:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.079467:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.079469:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.079471:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at 00000000412ab746. 00000001:00000010:3.0:1775764548.079602:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000c9a834a7. 00000001:00000010:3.0:1775764548.079609:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000d3385095. 00000001:00000010:3.0:1775764548.079614:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000d721b05e. 00000001:00000010:3.0:1775764548.079617:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000fb339ba9. 00000001:00000010:3.0:1775764548.079619:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000b2bf4094. 00000001:00000010:3.0:1775764548.079621:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000081d133fa. 00000020:00000010:3.0:1775764548.079625:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005d87781a. 00000020:00000010:3.0:1775764548.079630:0:6179:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.079634:0:6179:0:(tgt_main.c:757:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000067181d6f. 00000008:00000010:3.0:1775764548.079639:0:6179:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000a021b43c. 00020000:00000010:3.0:1775764548.079643:0:6179:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000003da9f5d9. 00008000:00000010:3.0:1775764548.079647:0:6179:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000004d2cc17. 00000080:00000010:3.0:1775764548.079652:0:6179:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000b5e74327. 00000020:00000010:3.0:1775764548.079658:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c2e1a443. 00000020:00000010:3.0:1775764548.079757:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000002aff1f92. 00000020:00000010:3.0:1775764548.079761:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000d8e4a5e3. 00000008:00000010:3.0:1775764548.079766:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000009b639a4e. 00020000:00000010:3.0:1775764548.079768:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000ed82c818. 00008000:00000010:3.0:1775764548.079770:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e44b074b. 00000080:00000010:3.0:1775764548.079774:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000012ff1347. 00000080:00000010:3.0:1775764548.079776:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000091504609. 00020000:00000010:3.0:1775764548.079779:0:6179:0:(lov_dev.c:599:lov_device_alloc()) kmalloced '(ld)': 632 at 0000000070499483. 00000020:00000010:3.0:1775764548.079793:0:6179:0:(lu_tgt_pool.c:52:lu_tgt_pool_init()) kmalloced '(op->op_array)': 8 at 0000000002745f36. 00020000:00000010:3.0:1775764548.080063:0:6179:0:(lov_dev.c:616:lov_device_alloc()) kmalloced '(ld->ld_md_tgts)': 4096 at 00000000a00d4790. 00020000:00000010:3.0:1775764548.080066:0:6179:0:(lov_dev.c:623:lov_device_alloc()) kmalloced '(ld->ld_lov->lov_mdc_tgts)': 4096 at 00000000a73e9959. 00000020:00000010:3.0:1775764548.080107:0:6179:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at 000000009a203707. 00000020:00000010:3.0:1775764548.080124:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000ab4958f9. 00000020:00000010:3.0:1775764548.080127:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at 00000000b1d095f5. 00000020:00000010:3.0:1775764548.080135:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a68b0f9d. 00000040:00000010:3.0:1775764548.080139:0:6179:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000da89af52. 00000020:00000010:3.0:1775764548.080144:0:6179:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000061ba7c40. 00000020:00000010:3.0:1775764548.080149:0:6179:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000083ecdf15. 00000020:00000010:3.0:1775764548.080154:0:6179:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000a4fb9452. 80000000:00000010:3.0:1775764548.080158:0:6179:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000b7f4c198. 40000000:00000010:3.0:1775764548.080162:0:6179:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 0000000042d60ad6. 00000008:00000010:3.0:1775764548.080166:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000010bab7b9. 00020000:00000010:3.0:1775764548.080168:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000aa880e73. 00008000:00000010:3.0:1775764548.080170:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000007f7b4538. 00100000:00000010:3.0:1775764548.080173:0:6179:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000298deaca. 00040000:00000010:3.0:1775764548.080177:0:6179:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000f0ca3a1d. 00040000:00000010:3.0:1775764548.080181:0:6179:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000097be601e. 00040000:00000010:3.0:1775764548.080184:0:6179:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000009ba2065. 00000004:00000010:3.0:1775764548.080196:0:6179:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000c019ca24. 00000004:00000010:3.0:1775764548.080200:0:6179:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 00000000fcaa40fc. 00000004:00000010:3.0:1775764548.080204:0:6179:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000006c838b67. 00002000:00000010:3.0:1775764548.080207:0:6179:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000e1120030. 00000004:00000010:3.0:1775764548.080210:0:6179:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 0000000018edd4d1. 00000080:00000010:3.0:1775764548.080212:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000065c599c2. 00000080:00000010:3.0:1775764548.080214:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000b1df9bd8. 00000020:00000010:3.0:1775764548.080216:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000074dd89e4. 00000040:00000010:3.0:1775764548.080218:0:6179:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000df020cbc. 00000020:00000010:3.0:1775764548.080220:0:6179:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000002aea7bb4. 00000020:00000010:3.0:1775764548.080222:0:6179:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 000000004a5374a0. 00000020:00000010:3.0:1775764548.080223:0:6179:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000580903b4. 80000000:00000010:3.0:1775764548.080225:0:6179:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000ed267393. 40000000:00000010:3.0:1775764548.080229:0:6179:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000003d150e87. 00000008:00000010:3.0:1775764548.080231:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000dd3cd90b. 00020000:00000010:3.0:1775764548.080232:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000cc32bb9c. 00008000:00000010:3.0:1775764548.080234:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000038668746. 00100000:00000010:3.0:1775764548.080236:0:6179:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000da839078. 00040000:00000010:3.0:1775764548.080237:0:6179:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000006230525c. 00040000:00000010:3.0:1775764548.080238:0:6179:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000091340f42. 00040000:00000010:3.0:1775764548.080240:0:6179:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000777d7d11. 00000004:00000010:3.0:1775764548.080241:0:6179:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000e2719968. 00000004:00000010:3.0:1775764548.080243:0:6179:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 000000002058afdb. 00000004:00000010:3.0:1775764548.080245:0:6179:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000ed4ccf58. 00002000:00000010:3.0:1775764548.080246:0:6179:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000a76fd6ce. 00000004:00000010:3.0:1775764548.080247:0:6179:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 000000002fda445a. 00000080:00000010:3.0:1775764548.080248:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000007e4207c2. 00000080:00000010:3.0:1775764548.080250:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000903812a7. 00000020:00000010:3.0:1775764548.080253:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000002aff1f92. 00000020:00000010:3.0:1775764548.080256:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000d8e4a5e3. 00000008:00000010:3.0:1775764548.080259:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000009b639a4e. 00020000:00000010:3.0:1775764548.080261:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000ed82c818. 00008000:00000010:3.0:1775764548.080263:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000e44b074b. 00000080:00000010:3.0:1775764548.080265:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000012ff1347. 00000080:00000010:3.0:1775764548.080267:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 0000000091504609. 00000020:00000010:3.0:1775764548.080269:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c2e1a443. 00000020:00000010:3.0:1775764548.080272:0:6179:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.080274:0:6179:0:(tgt_main.c:769:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000067181d6f. 00000008:00000010:3.0:1775764548.080276:0:6179:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000a021b43c. 00020000:00000010:3.0:1775764548.080278:0:6179:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000003da9f5d9. 00008000:00000010:3.0:1775764548.080280:0:6179:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000004d2cc17. 00000080:00000010:3.0:1775764548.080281:0:6179:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000b5e74327. 00000020:00000010:3.0:1775764548.080283:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005d87781a. 00000020:00000010:3.0:1775764548.080287:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 160 at 00000000412ab746. 00000020:00000010:3.0:1775764548.080290:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.080296:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000b2c32b4c. 00000020:00000010:3.0:1775764548.080303:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000b2c32b4c. 00000020:00000010:3.0:1775764548.080308:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000b2c32b4c. 00000020:00000010:3.0:1775764548.080310:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000b2c32b4c. 00000020:00000010:3.0:1775764548.080313:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.080316:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.080322:0:6179:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6592 at 00000000cb43bcb4. 00000020:00000010:3.0:1775764548.080326:0:6179:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 0000000085eaf097. 00000020:00000010:3.0:1775764548.080338:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.080341:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.080344:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.080346:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at 00000000412ab746. 00000001:00000010:3.0:1775764548.080360:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000b2c32b4c. 00000001:00000010:3.0:1775764548.080362:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000000f779cc8. 00000001:00000010:3.0:1775764548.080364:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000e7c7201f. 00000001:00000010:3.0:1775764548.080366:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000005d87781a. 00000001:00000010:3.0:1775764548.080367:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c2e1a443. 00000001:00000010:3.0:1775764548.080368:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000002aff1f92. 00000020:00000010:3.0:1775764548.080371:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b5d79eeb. 00000020:00000010:3.0:1775764548.080373:0:6179:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.080374:0:6179:0:(tgt_main.c:757:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000067181d6f. 00000008:00000010:3.0:1775764548.080376:0:6179:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000a021b43c. 00020000:00000010:3.0:1775764548.080377:0:6179:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000003da9f5d9. 00008000:00000010:3.0:1775764548.080379:0:6179:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000004d2cc17. 00000080:00000010:3.0:1775764548.080381:0:6179:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000b5e74327. 00000020:00000010:3.0:1775764548.080383:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004636cd07. 00000020:00000010:3.0:1775764548.080385:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000007904fda4. 00000040:00000010:3.0:1775764548.080387:0:6179:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000df952384. 00000020:00000010:3.0:1775764548.080389:0:6179:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000ed1cd92a. 00000020:00000010:3.0:1775764548.080391:0:6179:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000008a500997. 00040000:00000010:3.0:1775764548.080393:0:6179:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000002abfc754. 00040000:00000010:3.0:1775764548.080395:0:6179:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000074a32aad. 00800000:00000010:3.0:1775764548.080401:0:6179:0:(lmv_obd.c:1222:lmv_device_alloc()) kmalloced '(lu)': 64 at 0000000058922e06. 00000020:00000010:3.0:1775764548.082151:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000b1273bc1. 00000020:00000010:3.0:1775764548.082162:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 00000000752f68c4. 80000000:00000010:3.0:1775764548.082182:0:6179:0:(fld_cache.c:47:fld_cache_init()) kmalloced '(cache)': 168 at 0000000049cbbb3a. 00800000:00000010:3.0:1775764548.082221:0:6179:0:(lmv_obd.c:1282:lmv_device_alloc()) kmalloced '(pat)': 280 at 00000000fb5feae4. 00800000:00000010:3.0:1775764548.082223:0:6179:0:(lmv_obd.c:1291:lmv_device_alloc()) kmalloced '(pat)': 280 at 00000000a7f6907b. 00000020:00000010:3.0:1775764548.082228:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000007904fda4. 00000040:00000010:3.0:1775764548.082233:0:6179:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000df952384. 00000020:00000010:3.0:1775764548.082236:0:6179:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000ed1cd92a. 00000020:00000010:3.0:1775764548.082240:0:6179:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 000000008a500997. 00040000:00000010:3.0:1775764548.082245:0:6179:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 000000002abfc754. 00040000:00000010:3.0:1775764548.082250:0:6179:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 0000000074a32aad. 00000020:00000010:3.0:1775764548.082252:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004636cd07. 00000020:00000010:3.0:1775764548.082259:0:6179:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.082262:0:6179:0:(tgt_main.c:769:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000067181d6f. 00000008:00000010:3.0:1775764548.082266:0:6179:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000a021b43c. 00020000:00000010:3.0:1775764548.082268:0:6179:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000003da9f5d9. 00008000:00000010:3.0:1775764548.082272:0:6179:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000004d2cc17. 00000080:00000010:3.0:1775764548.082275:0:6179:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000b5e74327. 00000020:00000010:3.0:1775764548.082277:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b5d79eeb. 00000020:00000010:3.0:1775764548.082283:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 160 at 00000000412ab746. 00000020:00000010:3.0:1775764548.082286:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.082302:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082306:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082312:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082313:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082317:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 96 at 00000000d8e4a5e3. 00000020:00000010:3.0:1775764548.082321:0:6179:0:(obd_config.c:1148:class_add_profile()) kmalloced '(lprof)': 48 at 00000000e51b4ac5. 00000020:00000010:3.0:1775764548.082324:0:6179:0:(obd_config.c:1154:class_add_profile()) kmalloced '(lprof->lp_profile)': 14 at 00000000a4b35eb4. 00000020:00000010:3.0:1775764548.082327:0:6179:0:(obd_config.c:1160:class_add_profile()) kmalloced '(lprof->lp_dt)': 14 at 000000008afd3e8c. 00000020:00000010:3.0:1775764548.082329:0:6179:0:(obd_config.c:1167:class_add_profile()) kmalloced '(lprof->lp_md)': 14 at 000000002726bff8. 00000020:00000010:3.0:1775764548.082331:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 96 at 00000000d8e4a5e3. 00000020:00000010:3.0:1775764548.082340:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082342:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082347:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082349:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000002abfc754. 00000020:00000010:3.0:1775764548.082351:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 00000000fbfd1a13. 00000100:00000010:3.0:1775764548.082356:0:6179:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 00000000b72140ef. 00000400:00000200:3.0:1775764548.082362:0:6179:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1775764548.082369:0:6179:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.082375:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.082380:0:6179:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1775764548.082384:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.082389:0:6179:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1775764548.082392:0:6179:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:3.0:1775764548.082393:0:6179:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 00000100:00000010:3.0:1775764548.082395:0:6179:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 00000000b72140ef. 00000020:00000010:3.0:1775764548.082398:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 00000000fbfd1a13. 00000020:00000010:3.0:1775764548.082400:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000fbfd1a13. 00000020:00000010:3.0:1775764548.082403:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000412ab746. 00000020:00000010:3.0:1775764548.082413:0:6179:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6592 at 000000008c5854d9. 00000020:00000010:3.0:1775764548.082417:0:6179:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000ed1cd92a. 00000020:00000010:3.0:1775764548.082430:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000412ab746. 00000020:00000010:3.0:1775764548.082431:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000fbfd1a13. 00000020:00000010:3.0:1775764548.082433:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000fbfd1a13. 00000020:00000010:3.0:1775764548.082435:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000412ab746. 00000001:00000010:3.0:1775764548.082459:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000002abfc754. 00000001:00000010:3.0:1775764548.082463:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 0000000043dcadd7. 00000001:00000010:3.0:1775764548.082465:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000006ee22bc. 00000001:00000010:3.0:1775764548.082467:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000167e28b2. 00000001:00000010:3.0:1775764548.082468:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000e1257957. 00000001:00000010:3.0:1775764548.082470:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000afe2ca58. 00000020:00000010:3.0:1775764548.082473:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c0650cbc. 00000020:00000010:3.0:1775764548.082476:0:6179:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000194f3f46. 00000020:00000010:3.0:1775764548.082481:0:6179:0:(tgt_main.c:757:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000067181d6f. 00000008:00000010:2.0:1775764548.088041:0:6179:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000534476aa. 00020000:00000010:2.0:1775764548.088062:0:6179:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000b3134aae. 00008000:00000010:2.0:1775764548.088087:0:6179:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000004507b62e. 00000080:00000010:2.0:1775764548.088100:0:6179:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000e80a22a9. 00000020:00000010:2.0:1775764548.088109:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f8d13b4. 00000020:00000010:2.0:1775764548.088118:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000dcc71704. 00000020:00000010:2.0:1775764548.088122:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000009e1b72dc. 00000008:00000010:2.0:1775764548.088128:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000498fc7eb. 00020000:00000010:2.0:1775764548.088132:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000dc58430a. 00008000:00000010:2.0:1775764548.088136:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000030c06f1. 00000080:00000010:2.0:1775764548.088140:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000004f2134c6. 00000080:00000010:2.0:1775764548.088143:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000bd63f9c8. 00000002:00000010:2.0:1775764548.088148:0:6179:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at 00000000a06dada0. 00010000:00000010:2.0:1775764548.088169:0:6179:0:(ldlm_lib.c:420:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000df2008c0. 00010000:00000010:2.0:1775764548.088174:0:6179:0:(ldlm_lib.c:433:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000008dc9e257. 00010000:00000010:2.0:1775764548.088178:0:6179:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at 000000001b6271e7. 00000020:00000010:2.0:1775764548.088182:0:6179:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 00000000ce7f7993. 00000100:00000200:2.0:1775764548.088203:0:6179:0:(events.c:559:ptlrpc_uuid_to_peer()) 192.168.202.104@tcp->12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.088210:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.088215:0:6179:0:(peer.c:1480:LNetPrimaryNID()) NID 192.168.202.104@tcp primary NID 192.168.202.104@tcp rc 0 00010000:00000010:2.0:1775764548.088219:0:6179:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000d614c892. 00010000:00000010:2.0:1775764548.088223:0:6179:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000a3f93b20. 00000001:00000010:2.0:1775764548.088227:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 0000000077b944d6. 00000001:00000010:2.0:1775764548.088231:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at 0000000073a64baf. 00000001:00000010:2.0:1775764548.088274:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000073022dc8. 00000001:00000010:2.0:1775764548.088316:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000960708e7. 00000001:00000010:2.0:1775764548.088360:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000930c3620. 00000001:00000010:2.0:1775764548.088395:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000008fcd8018. 00000001:00000010:2.0:1775764548.088427:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000d8c89041. 00000001:00000010:2.0:1775764548.088459:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000022839cef. 00000001:00000010:2.0:1775764548.088510:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000031405f97. 00000001:00000010:2.0:1775764548.088543:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000066ad8322. 00000001:00000010:2.0:1775764548.088587:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000008d3ecf16. 00000001:00000010:2.0:1775764548.088629:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000006a1c0b82. 00000001:00000010:2.0:1775764548.088746:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e3f5d061. 00000001:00000010:2.0:1775764548.088781:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000084558614. 00000001:00000010:2.0:1775764548.088901:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000004372cb9c. 00000001:00000010:2.0:1775764548.088947:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000101195bc. 00000001:00000010:2.0:1775764548.088989:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000092256f93. 00000001:00000010:2.0:1775764548.089035:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000b4487b89. 00000001:00000010:2.0:1775764548.089075:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000048221b5b. 00000001:00000010:2.0:1775764548.089125:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000008bec5081. 00000001:00000010:2.0:1775764548.089176:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000006384d699. 00000001:00000010:2.0:1775764548.089208:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000779ab08d. 00000001:00000010:2.0:1775764548.089241:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000005e0f2de7. 00000001:00000010:2.0:1775764548.089273:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e38842cf. 00000001:00000010:2.0:1775764548.089309:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000010df2e9a. 00000001:00000010:2.0:1775764548.089353:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c1b1b075. 00000001:00000010:2.0:1775764548.089386:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c59197d6. 00000001:00000010:2.0:1775764548.089417:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000002c47bdf4. 00000001:00000010:2.0:1775764548.089454:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c390826b. 00000001:00000010:2.0:1775764548.089512:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c656f4ce. 00000001:00000010:2.0:1775764548.089555:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000057ea79a. 00000001:00000010:2.0:1775764548.089621:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000014a54470. 00000001:00000010:2.0:1775764548.090531:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000bcaf1354. 00000001:00000010:2.0:1775764548.090601:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000057aeec50. 00000020:00000010:2.0:1775764548.091961:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000f29b6b28. 00000020:00000010:2.0:1775764548.091973:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 000000004e36c66a. 00000020:00000010:2.0:1775764548.092070:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000872fa80f. 00000020:00000010:2.0:1775764548.092074:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 00000000e4435b0e. 00000020:00000010:2.0:1775764548.092325:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000faac6640. 00000020:00000010:2.0:1775764548.092328:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 0000000005f94bf5. 00000020:00000010:2.0:1775764548.092356:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000041bdaa88. 00000020:00000010:2.0:1775764548.092359:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 000000003a6ef60a. 00000040:00000010:2.0:1775764548.092383:0:6179:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 0000000090526115. 00000002:00000010:2.0:1775764548.092393:0:6179:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at 0000000006625522. 00000020:00000010:2.0:1775764548.095220:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000dcc71704. 00000020:00000010:2.0:1775764548.095232:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 000000009e1b72dc. 00000008:00000010:2.0:1775764548.095239:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000498fc7eb. 00020000:00000010:2.0:1775764548.095248:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000dc58430a. 00008000:00000010:2.0:1775764548.095254:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000030c06f1. 00000080:00000010:2.0:1775764548.095261:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000004f2134c6. 00000080:00000010:2.0:1775764548.095265:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000bd63f9c8. 00000020:00000010:2.0:1775764548.095270:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f8d13b4. 00000020:00000010:2.0:1775764548.095278:0:6179:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000194f3f46. 00000020:00000010:2.0:1775764548.095284:0:6179:0:(tgt_main.c:769:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000067181d6f. 00000008:00000010:2.0:1775764548.095291:0:6179:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000534476aa. 00020000:00000010:2.0:1775764548.095295:0:6179:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 00000000b3134aae. 00008000:00000010:2.0:1775764548.095299:0:6179:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000004507b62e. 00000080:00000010:2.0:1775764548.095303:0:6179:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000e80a22a9. 00000020:00000010:2.0:1775764548.095306:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c0650cbc. 00000020:00000010:2.0:1775764548.095313:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000412ab746. 00000020:00000010:2.0:1775764548.095317:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000fbfd1a13. 00000020:00000010:2.0:1775764548.095329:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000506b983f. 00000020:00000010:2.0:1775764548.095335:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at 00000000b684c81e. 00000020:00000010:2.0:1775764548.095345:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f8d13b4. 00000020:00000010:2.0:1775764548.095349:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000dcc71704. 00000040:00000010:2.0:1775764548.095354:0:6179:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000002bd1e81a. 00000020:00000010:2.0:1775764548.095359:0:6179:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000009761ac83. 00000020:00000010:2.0:1775764548.095363:0:6179:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000f7d67d8e. 00040000:00000010:2.0:1775764548.095370:0:6179:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000001e882c7e. 00040000:00000010:2.0:1775764548.095374:0:6179:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000fa88afa1. 00800000:00000010:2.0:1775764548.095389:0:6179:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at 00000000e5156aba. 00020000:00000010:2.0:1775764548.095395:0:6179:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at 00000000c801a8b2. 00000020:00000010:2.0:1775764548.095398:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000dcc71704. 00000040:00000010:2.0:1775764548.095401:0:6179:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 000000002bd1e81a. 00000020:00000010:2.0:1775764548.095404:0:6179:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 000000009761ac83. 00000020:00000010:2.0:1775764548.095409:0:6179:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000f7d67d8e. 00040000:00000010:2.0:1775764548.095419:0:6179:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 000000001e882c7e. 00040000:00000010:2.0:1775764548.095423:0:6179:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000fa88afa1. 00000020:00000010:2.0:1775764548.095425:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f8d13b4. 00000020:00000010:2.0:1775764548.095430:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f8d13b4. 00000020:00000010:2.0:1775764548.095432:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000fa88afa1. 00000020:00000010:2.0:1775764548.095436:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000596fe25f. 00000008:00000010:2.0:1775764548.095440:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000498fc7eb. 00020000:00000010:2.0:1775764548.095443:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000dc58430a. 00008000:00000010:2.0:1775764548.095447:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000030c06f1. 00000080:00000010:2.0:1775764548.095451:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000004f2134c6. 00000080:00000010:2.0:1775764548.095454:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000bd63f9c8. 00000020:00000010:2.0:1775764548.095465:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000fa88afa1. 00000020:00000010:2.0:1775764548.095467:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000596fe25f. 00000008:00000010:2.0:1775764548.095470:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000498fc7eb. 00020000:00000010:2.0:1775764548.095472:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000dc58430a. 00008000:00000010:2.0:1775764548.095474:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000030c06f1. 00000080:00000010:2.0:1775764548.095476:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000004f2134c6. 00000080:00000010:2.0:1775764548.095478:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000bd63f9c8. 00000020:00000010:2.0:1775764548.095480:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000009f8d13b4. 00000020:00000010:2.0:1775764548.095506:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 152 at 00000000b684c81e. 00000020:00000010:2.0:1775764548.095509:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000506b983f. 00000020:00000010:2.0:1775764548.095515:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000001e882c7e. 00000020:00000010:2.0:1775764548.095517:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000001e882c7e. 00000020:00000010:2.0:1775764548.095522:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000001e882c7e. 00000020:00000010:2.0:1775764548.095524:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000001e882c7e. 00000020:00000010:2.0:1775764548.095530:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000001e882c7e. 00000020:00000010:2.0:1775764548.095531:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000001e882c7e. 00000020:00000010:2.0:1775764548.095534:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 00000000506b983f. 00000100:00000010:2.0:1775764548.095539:0:6179:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 0000000028dff29d. 00000400:00000200:2.0:1775764548.095544:0:6179:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:2.0:1775764548.095553:0:6179:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:2.0:1775764548.095562:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.095569:0:6179:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:2.0:1775764548.095582:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.095591:0:6179:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1775764548.095594:0:6179:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:2.0:1775764548.095597:0:6179:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 00000100:00000010:2.0:1775764548.095600:0:6179:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 0000000028dff29d. 00000020:00000010:2.0:1775764548.095604:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 00000000506b983f. 00000020:00000010:2.0:1775764548.095608:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000506b983f. 00000020:00000010:2.0:1775764548.095612:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000b684c81e. 00000020:00000010:2.0:1775764548.095638:0:6179:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6592 at 000000009a2d8673. 00000020:00000010:2.0:1775764548.095647:0:6179:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000009761ac83. 00000020:00000010:2.0:1775764548.097898:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000b684c81e. 00000020:00000010:2.0:1775764548.097910:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000506b983f. 00000020:00000010:2.0:1775764548.097924:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000506b983f. 00000020:00000010:2.0:1775764548.097930:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000b684c81e. 00000001:00000010:2.0:1775764548.097962:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000001e882c7e. 00000001:00000010:2.0:1775764548.097970:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000007c6b4bef. 00000001:00000010:2.0:1775764548.097975:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000009f8d13b4. 00000001:00000010:2.0:1775764548.097978:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000fa88afa1. 00000001:00000010:2.0:1775764548.097980:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000f7d67d8e. 00000001:00000010:2.0:1775764548.097983:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000002bd1e81a. 00000020:00000010:2.0:1775764548.097988:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dcc71704. 00000020:00000010:2.0:1775764548.097994:0:6179:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000134ac13c. 00000020:00000010:2.0:1775764548.098015:0:6179:0:(tgt_main.c:757:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000efff7b46. 00000008:00000010:2.0:1775764548.098024:0:6179:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000534476aa. 00020000:00000010:2.0:1775764548.098029:0:6179:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000b3134aae. 00008000:00000010:2.0:1775764548.098034:0:6179:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000004507b62e. 00000080:00000010:2.0:1775764548.098042:0:6179:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000e80a22a9. 00000020:00000010:2.0:1775764548.098046:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a7135789. 00000020:00000010:2.0:1775764548.098050:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b5f5bca5. 00000020:00000010:2.0:1775764548.098054:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000596fe25f. 00000008:00000010:2.0:1775764548.098059:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000498fc7eb. 00020000:00000010:2.0:1775764548.098062:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000dc58430a. 00008000:00000010:2.0:1775764548.098071:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000030c06f1. 00000080:00000010:2.0:1775764548.098075:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000004f2134c6. 00000080:00000010:2.0:1775764548.098078:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000bd63f9c8. 00000002:00000010:2.0:1775764548.098083:0:6179:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at 0000000078228d42. 00010000:00000010:2.0:1775764548.098100:0:6179:0:(ldlm_lib.c:420:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000e3aeac15. 00010000:00000010:2.0:1775764548.098106:0:6179:0:(ldlm_lib.c:433:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000005bd12a77. 00010000:00000010:2.0:1775764548.098111:0:6179:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at 000000000a8849a0. 00000020:00000010:2.0:1775764548.098128:0:6179:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 00000000ac986517. 00000100:00000200:2.0:1775764548.098158:0:6179:0:(events.c:559:ptlrpc_uuid_to_peer()) 192.168.202.104@tcp->12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.098166:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.098172:0:6179:0:(peer.c:1480:LNetPrimaryNID()) NID 192.168.202.104@tcp primary NID 192.168.202.104@tcp rc 0 00010000:00000010:2.0:1775764548.098178:0:6179:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000cb7e16d1. 00010000:00000010:2.0:1775764548.098184:0:6179:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000b0a4b29b. 00000001:00000010:2.0:1775764548.098187:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 0000000020f07bf6. 00000001:00000010:2.0:1775764548.098191:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at 000000001ce83172. 00000001:00000010:2.0:1775764548.098235:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a1e1dafe. 00000001:00000010:2.0:1775764548.098282:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000fc6f6aea. 00000001:00000010:2.0:1775764548.098333:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000005a83b98e. 00000001:00000010:2.0:1775764548.098377:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000007c93fc78. 00000001:00000010:2.0:1775764548.098425:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000002d8d7502. 00000001:00000010:2.0:1775764548.098469:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000008327b1be. 00000001:00000010:2.0:1775764548.098538:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000083d7cd02. 00000001:00000010:2.0:1775764548.098581:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000efee0598. 00000001:00000010:2.0:1775764548.098625:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000920ddc34. 00000001:00000010:2.0:1775764548.099566:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e17e1a46. 00000001:00000010:2.0:1775764548.099622:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000039145984. 00000001:00000010:2.0:1775764548.099655:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000f74a9285. 00000001:00000010:2.0:1775764548.099918:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000094ef931c. 00000001:00000010:2.0:1775764548.099964:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000762b95b7. 00000001:00000010:2.0:1775764548.100009:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000786a058a. 00000001:00000010:2.0:1775764548.100055:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000af69c3a0. 00000001:00000010:2.0:1775764548.100107:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a44af80f. 00000001:00000010:2.0:1775764548.100152:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000d448ec1f. 00000001:00000010:2.0:1775764548.100198:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000034258d57. 00000001:00000010:2.0:1775764548.100246:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000009ff92e32. 00000001:00000010:2.0:1775764548.100292:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c7cf8eb9. 00000001:00000010:2.0:1775764548.100337:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000d3ea69a2. 00000001:00000010:2.0:1775764548.100387:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e0621f6d. 00000001:00000010:2.0:1775764548.100430:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000001379bc58. 00000001:00000010:2.0:1775764548.101932:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c5433eee. 00000001:00000010:2.0:1775764548.101983:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000f512aa1f. 00000001:00000010:2.0:1775764548.102015:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000f969d903. 00000001:00000010:2.0:1775764548.102046:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000080a04e7d. 00000001:00000010:2.0:1775764548.102078:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000155383b5. 00000001:00000010:2.0:1775764548.102125:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000001a0e0707. 00000001:00000010:2.0:1775764548.102172:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000cd106ac1. 00000001:00000010:2.0:1775764548.102206:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a46c52c9. 00000020:00000010:2.0:1775764548.102279:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000ef132e4b. 00000020:00000010:2.0:1775764548.102283:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 00000000a7ed3e3c. 00000020:00000010:2.0:1775764548.102384:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000f4313bcd. 00000020:00000010:2.0:1775764548.102393:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 000000008e759fd6. 00000020:00000010:2.0:1775764548.103019:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000009f1ce530. 00000020:00000010:2.0:1775764548.103025:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 00000000983669ce. 00000020:00000010:2.0:1775764548.103046:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000e55b2944. 00000020:00000010:2.0:1775764548.103049:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 00000000baa93ca4. 00000040:00000010:2.0:1775764548.103072:0:6179:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 0000000062c432f2. 00000002:00000010:2.0:1775764548.103081:0:6179:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at 00000000dc3d65cd. 00000020:00000010:3.0:1775764548.105120:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000b5f5bca5. 00000020:00000010:3.0:1775764548.105130:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.105136:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000498fc7eb. 00020000:00000010:3.0:1775764548.105143:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000dc58430a. 00008000:00000010:3.0:1775764548.105148:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000030c06f1. 00000080:00000010:3.0:1775764548.105153:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000004f2134c6. 00000080:00000010:3.0:1775764548.105157:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000bd63f9c8. 00000020:00000010:3.0:1775764548.105160:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a7135789. 00000020:00000010:3.0:1775764548.105167:0:6179:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000134ac13c. 00000020:00000010:3.0:1775764548.105170:0:6179:0:(tgt_main.c:769:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000efff7b46. 00000008:00000010:3.0:1775764548.105175:0:6179:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000534476aa. 00020000:00000010:3.0:1775764548.105178:0:6179:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 00000000b3134aae. 00008000:00000010:3.0:1775764548.105181:0:6179:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000004507b62e. 00000080:00000010:3.0:1775764548.105184:0:6179:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000e80a22a9. 00000020:00000010:3.0:1775764548.105187:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000dcc71704. 00000020:00000010:3.0:1775764548.105193:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000b684c81e. 00000020:00000010:3.0:1775764548.105196:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000506b983f. 00000020:00000010:3.0:1775764548.105208:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000461881a5. 00000020:00000010:3.0:1775764548.105213:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at 00000000c410ab98. 00000020:00000010:3.0:1775764548.105223:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000fd3ffd85. 00000020:00000010:3.0:1775764548.105226:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000079a34f85. 00000040:00000010:3.0:1775764548.105230:0:6179:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000008e4b6617. 00000020:00000010:3.0:1775764548.105234:0:6179:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000976a5265. 00000020:00000010:3.0:1775764548.105238:0:6179:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000ce920b81. 00040000:00000010:3.0:1775764548.105244:0:6179:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000e15767c0. 00040000:00000010:3.0:1775764548.105247:0:6179:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000bebbb807. 00800000:00000010:3.0:1775764548.105261:0:6179:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at 00000000d93383d8. 00000020:00000010:3.0:1775764548.105264:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000079a34f85. 00000040:00000010:3.0:1775764548.105267:0:6179:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 000000008e4b6617. 00000020:00000010:3.0:1775764548.105269:0:6179:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000976a5265. 00000020:00000010:3.0:1775764548.105272:0:6179:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000ce920b81. 00040000:00000010:3.0:1775764548.105281:0:6179:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 00000000e15767c0. 00040000:00000010:3.0:1775764548.105286:0:6179:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000bebbb807. 00000020:00000010:3.0:1775764548.105288:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000fd3ffd85. 00000020:00000010:3.0:1775764548.105293:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000fd3ffd85. 00000020:00000010:3.0:1775764548.105295:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000bebbb807. 00000020:00000010:3.0:1775764548.105298:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000d8e4a5e3. 00000008:00000010:3.0:1775764548.105303:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000009b639a4e. 00020000:00000010:3.0:1775764548.105306:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000ed82c818. 00008000:00000010:3.0:1775764548.105310:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e44b074b. 00000080:00000010:3.0:1775764548.105314:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000012ff1347. 00000080:00000010:3.0:1775764548.105317:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000091504609. 00000020:00000010:3.0:1775764548.105324:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000bebbb807. 00000020:00000010:3.0:1775764548.105326:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000d8e4a5e3. 00000008:00000010:3.0:1775764548.105329:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000009b639a4e. 00020000:00000010:3.0:1775764548.105331:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000ed82c818. 00008000:00000010:3.0:1775764548.105333:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000e44b074b. 00000080:00000010:3.0:1775764548.105336:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000012ff1347. 00000080:00000010:3.0:1775764548.105338:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 0000000091504609. 00000020:00000010:3.0:1775764548.105340:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000fd3ffd85. 00000020:00000010:3.0:1775764548.105342:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 152 at 00000000c410ab98. 00000020:00000010:3.0:1775764548.105344:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000461881a5. 00000020:00000010:3.0:1775764548.105350:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000e15767c0. 00000020:00000010:3.0:1775764548.105352:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000e15767c0. 00000020:00000010:3.0:1775764548.105357:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000e15767c0. 00000020:00000010:3.0:1775764548.105359:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000e15767c0. 00000020:00000010:3.0:1775764548.105365:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000e15767c0. 00000020:00000010:3.0:1775764548.105367:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000e15767c0. 00000020:00000010:3.0:1775764548.105370:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 00000000461881a5. 00000100:00000010:3.0:1775764548.105375:0:6179:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 000000000dfdfe8d. 00000400:00000200:3.0:1775764548.105381:0:6179:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1775764548.105389:0:6179:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.105397:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.105403:0:6179:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1775764548.105415:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.105422:0:6179:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1775764548.105425:0:6179:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:3.0:1775764548.105427:0:6179:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 00000100:00000010:3.0:1775764548.105429:0:6179:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 000000000dfdfe8d. 00000020:00000010:3.0:1775764548.105432:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 00000000461881a5. 00000020:00000010:3.0:1775764548.105436:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000461881a5. 00000020:00000010:3.0:1775764548.105439:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000c410ab98. 00000020:00000010:3.0:1775764548.105459:0:6179:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6592 at 00000000e3bbcdd2. 00000020:00000010:3.0:1775764548.105466:0:6179:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000976a5265. 00000020:00000010:3.0:1775764548.105511:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000c410ab98. 00000020:00000010:3.0:1775764548.105515:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000461881a5. 00000020:00000010:3.0:1775764548.105519:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000461881a5. 00000020:00000010:3.0:1775764548.105521:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000c410ab98. 00000001:00000010:3.0:1775764548.105544:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000e15767c0. 00000001:00000010:3.0:1775764548.105550:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000006b454fbd. 00000001:00000010:3.0:1775764548.105553:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000fd3ffd85. 00000001:00000010:3.0:1775764548.105555:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000bebbb807. 00000001:00000010:3.0:1775764548.105556:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ce920b81. 00000001:00000010:3.0:1775764548.105558:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000008e4b6617. 00000020:00000010:3.0:1775764548.105561:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000079a34f85. 00000020:00000010:3.0:1775764548.105564:0:6179:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 0000000099e7fc32. 00000020:00000010:3.0:1775764548.105568:0:6179:0:(tgt_main.c:757:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000018abeff9. 00000008:00000010:3.0:1775764548.105571:0:6179:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000a021b43c. 00020000:00000010:3.0:1775764548.105574:0:6179:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000003da9f5d9. 00008000:00000010:3.0:1775764548.105577:0:6179:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000004d2cc17. 00000080:00000010:3.0:1775764548.105580:0:6179:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000b5e74327. 00000020:00000010:3.0:1775764548.105582:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000aed1f6a4. 00000020:00000010:3.0:1775764548.105585:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c0650cbc. 00000020:00000010:3.0:1775764548.105587:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000d8e4a5e3. 00000008:00000010:3.0:1775764548.105589:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000009b639a4e. 00020000:00000010:3.0:1775764548.105591:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000ed82c818. 00008000:00000010:3.0:1775764548.105598:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e44b074b. 00000080:00000010:3.0:1775764548.105601:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000012ff1347. 00000080:00000010:3.0:1775764548.105603:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000091504609. 00000008:00000010:3.0:1775764548.105606:0:6179:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at 00000000665efb9c. 00010000:00000010:3.0:1775764548.105616:0:6179:0:(ldlm_lib.c:420:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000bf5ef302. 00010000:00000010:3.0:1775764548.105621:0:6179:0:(ldlm_lib.c:433:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000000205c366. 00000020:00000010:3.0:1775764548.105632:0:6179:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 00000000330e679f. 00000100:00000200:3.0:1775764548.105647:0:6179:0:(events.c:559:ptlrpc_uuid_to_peer()) 192.168.202.104@tcp->12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764548.105652:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.105655:0:6179:0:(peer.c:1480:LNetPrimaryNID()) NID 192.168.202.104@tcp primary NID 192.168.202.104@tcp rc 0 00010000:00000010:3.0:1775764548.105659:0:6179:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 000000005aa40a41. 00010000:00000010:3.0:1775764548.105673:0:6179:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000070c830d. 00000001:00000010:3.0:1775764548.105677:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000af3b8e3b. 00000001:00000010:3.0:1775764548.105680:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at 00000000aadb5f3f. 00000001:00000010:3.0:1775764548.105684:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000e4803ed9. 00000001:00000010:3.0:1775764548.105689:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000451f81d4. 00000001:00000010:3.0:1775764548.105695:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000d121d32e. 00000001:00000010:3.0:1775764548.105700:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000f78436b5. 00000001:00000010:3.0:1775764548.105705:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c17559af. 00000001:00000010:3.0:1775764548.105718:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000fb48b0b1. 00000001:00000010:3.0:1775764548.105723:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000de647d7a. 00000001:00000010:3.0:1775764548.105729:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000563f366a. 00000001:00000010:3.0:1775764548.105734:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000008c5e4607. 00000001:00000010:3.0:1775764548.105746:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000b6546532. 00000001:00000010:3.0:1775764548.105752:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000065a8f7a4. 00000001:00000010:3.0:1775764548.105757:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000004db6a0c5. 00000001:00000010:3.0:1775764548.105762:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000ebf42104. 00000001:00000010:3.0:1775764548.105777:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000df04b21e. 00000001:00000010:3.0:1775764548.105782:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000032ab34d2. 00000001:00000010:3.0:1775764548.105786:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c6b96a97. 00000020:00000010:3.0:1775764548.105865:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000913c61cf. 00000020:00000010:3.0:1775764548.105876:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 000000000319e854. 00000020:00000010:3.0:1775764548.105991:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000fe041237. 00000020:00000010:3.0:1775764548.105993:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 00000000dcc71704. 00000020:00000010:1.0:1775764548.107272:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000048ca094. 00000020:00000010:1.0:1775764548.107281:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 0000000065596ca4. 00000100:00000010:1.0:1775764548.107335:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000004cf6ba48. 00000100:00000010:1.0:1775764548.107634:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000007835a58a. 00000100:00000010:1.0:1775764548.107640:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ce8fb420. 00000100:00000010:1.0:1775764548.108481:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000e9bfbabe. 00000100:00000010:1.0:1775764548.108518:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000b0ce65d5. 00000100:00000010:1.0:1775764548.108905:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 0000000018762db4. 00000100:00000010:1.0:1775764548.108910:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000094d8448b. 00000100:00000010:1.0:1775764548.109231:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000006b1ea231. 00000100:00000010:1.0:1775764548.109242:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fb468d0f. 00000100:00000010:1.0:1775764548.109781:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000e0d58430. 00000100:00000010:1.0:1775764548.109786:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006ce9d824. 00000100:00000010:1.0:1775764548.110034:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000004d27d818. 00000100:00000010:1.0:1775764548.110038:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d15434db. 00000100:00000010:1.0:1775764548.110223:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000d316b459. 00000100:00000010:1.0:1775764548.110228:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000010f6a320. 00000100:00000010:1.0:1775764548.110814:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000470c5eea. 00000100:00000010:1.0:1775764548.110820:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000091cd915b. 00000100:00000010:1.0:1775764548.111153:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000186341f9. 00000100:00000010:1.0:1775764548.111158:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000437377f7. 00000100:00000010:1.0:1775764548.111481:0:6179:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000fbd4db38. 00000020:00000010:1.0:1775764548.111970:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000c0650cbc. 00000020:00000010:1.0:1775764548.111981:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000d8e4a5e3. 00000008:00000010:1.0:1775764548.111987:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000009b639a4e. 00020000:00000010:1.0:1775764548.111994:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000ed82c818. 00008000:00000010:1.0:1775764548.111999:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000e44b074b. 00000080:00000010:1.0:1775764548.112005:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000012ff1347. 00000080:00000010:1.0:1775764548.112011:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 0000000091504609. 00000020:00000010:1.0:1775764548.112014:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000aed1f6a4. 00000020:00000010:1.0:1775764548.112022:0:6179:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 0000000099e7fc32. 00000020:00000010:1.0:1775764548.112026:0:6179:0:(tgt_main.c:769:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000018abeff9. 00000008:00000010:1.0:1775764548.112031:0:6179:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000a021b43c. 00020000:00000010:1.0:1775764548.112035:0:6179:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000003da9f5d9. 00008000:00000010:1.0:1775764548.112040:0:6179:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000004d2cc17. 00000080:00000010:1.0:1775764548.112043:0:6179:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000b5e74327. 00000020:00000010:1.0:1775764548.112047:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000079a34f85. 00000020:00000010:1.0:1775764548.112054:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000c410ab98. 00000020:00000010:1.0:1775764548.112058:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000461881a5. 00000020:00000010:1.0:1775764548.112524:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 000000008491c33c. 00000020:00000010:1.0:1775764548.112532:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at 00000000cf9df0d9. 00000020:00000010:1.0:1775764548.112555:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d3115abc. 00000020:00000010:1.0:1775764548.112567:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e5c4e215. 00000020:00000010:1.0:1775764548.112573:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000dee0a3ca. 00000008:00000010:1.0:1775764548.112580:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000003049a7b2. 00020000:00000010:1.0:1775764548.112585:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000009e0a8aa. 00008000:00000010:1.0:1775764548.112590:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000cac43077. 00000080:00000010:1.0:1775764548.112597:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000016ac4737. 00000080:00000010:1.0:1775764548.112602:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000ac0ccfb2. 00020000:00000010:1.0:1775764548.112621:0:6179:0:(lov_obd.c:495:lov_add_target()) kmalloced '(tgt)': 368 at 0000000079a34f85. 00020000:00000010:1.0:1775764548.112628:0:6179:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at 000000009e089fb2. 00020000:00000010:1.0:1775764548.112636:0:6179:0:(lov_dev.c:357:lov_expand_targets()) kmalloced '(newd)': 512 at 00000000aed1f6a4. 00000020:00000010:1.0:1775764548.112640:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000e5c4e215. 00000020:00000010:1.0:1775764548.112643:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000dee0a3ca. 00000008:00000010:1.0:1775764548.112645:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000003049a7b2. 00020000:00000010:1.0:1775764548.112648:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 0000000009e0a8aa. 00008000:00000010:1.0:1775764548.112650:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000cac43077. 00000080:00000010:1.0:1775764548.112653:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000016ac4737. 00000080:00000010:1.0:1775764548.112656:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000ac0ccfb2. 00000020:00000010:1.0:1775764548.112657:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000d3115abc. 00000020:00000010:1.0:1775764548.112660:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 120 at 00000000cf9df0d9. 00000020:00000010:1.0:1775764548.112664:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 000000008491c33c. 00000020:00000010:1.0:1775764548.112671:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000097b8af74. 00000020:00000010:1.0:1775764548.112682:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000097b8af74. 00000020:00000010:1.0:1775764548.112689:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000097b8af74. 00000020:00000010:1.0:1775764548.112691:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000097b8af74. 00000020:00000010:1.0:1775764548.112697:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000097b8af74. 00000020:00000010:1.0:1775764548.112698:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000097b8af74. 00000020:00000010:1.0:1775764548.112701:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 000000008491c33c. 00000100:00000010:1.0:1775764548.112706:0:6179:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 00000000b3d9d090. 00000400:00000200:1.0:1775764548.112712:0:6179:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:1.0:1775764548.112722:0:6179:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:1.0:1775764548.112732:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.112739:0:6179:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1775764548.112752:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.113000:0:6179:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1775764548.113004:0:6179:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:1.0:1775764548.113007:0:6179:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 00000100:00000010:1.0:1775764548.113009:0:6179:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 00000000b3d9d090. 00000020:00000010:1.0:1775764548.113013:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 000000008491c33c. 00000020:00000010:1.0:1775764548.113018:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 000000008491c33c. 00000020:00000010:1.0:1775764548.113022:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000048b90b88. 00000020:00000010:1.0:1775764548.113048:0:6179:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6592 at 00000000d027b777. 00000020:00000010:1.0:1775764548.113056:0:6179:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000d228b7ec. 00000020:00000010:1.0:1775764548.113085:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000048b90b88. 00000020:00000010:1.0:1775764548.113088:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 000000008491c33c. 00000020:00000010:1.0:1775764548.113092:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 000000008491c33c. 00000020:00000010:1.0:1775764548.113095:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000048b90b88. 00000001:00000010:1.0:1775764548.113129:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 0000000097b8af74. 00000001:00000010:1.0:1775764548.113135:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 0000000034dfb72b. 00000001:00000010:1.0:1775764548.113139:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c0650cbc. 00000001:00000010:1.0:1775764548.113142:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000d3115abc. 00000001:00000010:1.0:1775764548.113144:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000e5c4e215. 00000001:00000010:1.0:1775764548.113147:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000092a59c0c. 00000020:00000010:1.0:1775764548.113151:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f777ccb4. 00000020:00000010:1.0:1775764548.113154:0:6179:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cf9df0d9. 00000020:00000010:1.0:1775764548.113166:0:6179:0:(tgt_main.c:757:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000e840a280. 00000008:00000010:1.0:1775764548.113179:0:6179:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000ed0ce56e. 00020000:00000010:1.0:1775764548.113193:0:6179:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000009a72f5f6. 00008000:00000010:1.0:1775764548.113212:0:6179:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 00000000eefcea5b. 00000080:00000010:1.0:1775764548.113224:0:6179:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000b81dcbdc. 00000020:00000010:1.0:1775764548.113229:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d7502ed5. 00000020:00000010:1.0:1775764548.113232:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b5040633. 00000020:00000010:1.0:1775764548.113235:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000dee0a3ca. 00000008:00000010:1.0:1775764548.113238:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000003049a7b2. 00020000:00000010:1.0:1775764548.113240:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000009e0a8aa. 00008000:00000010:1.0:1775764548.113245:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000cac43077. 00000080:00000010:1.0:1775764548.113249:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000016ac4737. 00000080:00000010:1.0:1775764548.113251:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000ac0ccfb2. 00000008:00000010:1.0:1775764548.113255:0:6179:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at 00000000b1ed53da. 00010000:00000010:1.0:1775764548.113268:0:6179:0:(ldlm_lib.c:420:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000e82b6e4e. 00010000:00000010:1.0:1775764548.113281:0:6179:0:(ldlm_lib.c:433:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 0000000028b531d5. 00000020:00000010:1.0:1775764548.113460:0:6179:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 00000000dfeaea0f. 00000100:00000200:1.0:1775764548.113477:0:6179:0:(events.c:559:ptlrpc_uuid_to_peer()) 192.168.202.104@tcp->12345-192.168.202.104@tcp 00000400:00000200:1.0:1775764548.113483:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.113520:0:6179:0:(peer.c:1480:LNetPrimaryNID()) NID 192.168.202.104@tcp primary NID 192.168.202.104@tcp rc 0 00010000:00000010:1.0:1775764548.113527:0:6179:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000ea6c510c. 00010000:00000010:1.0:1775764548.113534:0:6179:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 0000000010b91f19. 00000001:00000010:1.0:1775764548.113606:0:6179:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 000000006a02c4eb. 00000001:00000010:1.0:1775764548.113610:0:6179:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at 000000007f5da972. 00000001:00000010:1.0:1775764548.113616:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000035122612. 00000001:00000010:1.0:1775764548.113623:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000003b14f463. 00000001:00000010:1.0:1775764548.113635:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000007429bc02. 00000001:00000010:1.0:1775764548.113641:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c4ed65c0. 00000001:00000010:1.0:1775764548.113646:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000008c9a9e54. 00000001:00000010:1.0:1775764548.113651:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000408f6cfd. 00000001:00000010:1.0:1775764548.113662:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000008565865e. 00000001:00000010:1.0:1775764548.113668:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000031bd27e8. 00000001:00000010:1.0:1775764548.113751:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000a4edd3e6. 00000001:00000010:1.0:1775764548.113848:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000373c4cd7. 00000001:00000010:1.0:1775764548.113870:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c44dd4b9. 00000001:00000010:1.0:1775764548.113876:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000b0abcfa6. 00000001:00000010:1.0:1775764548.113882:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000f9240006. 00000001:00000010:1.0:1775764548.113887:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000ab1752dc. 00000001:00000010:1.0:1775764548.113899:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000006c3b4c76. 00000001:00000010:1.0:1775764548.113904:0:6179:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c3520a3a. 00000020:00000010:1.0:1775764548.113994:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000004931d38d. 00000020:00000010:1.0:1775764548.114003:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 00000000ba4ca6f5. 00000020:00000010:1.0:1775764548.114182:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000a8f2ac1e. 00000020:00000010:1.0:1775764548.114186:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 000000008092ea2f. 00000020:00000010:1.0:1775764548.116203:0:6179:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000042edd8c1. 00000020:00000010:1.0:1775764548.116212:0:6179:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 00000000ba3dd5da. 00000020:00000010:1.0:1775764548.116244:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000b5040633. 00000020:00000010:1.0:1775764548.116248:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000dee0a3ca. 00000008:00000010:1.0:1775764548.116253:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000003049a7b2. 00020000:00000010:1.0:1775764548.116258:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 0000000009e0a8aa. 00008000:00000010:1.0:1775764548.116262:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000cac43077. 00000080:00000010:1.0:1775764548.116267:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000016ac4737. 00000080:00000010:1.0:1775764548.116270:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000ac0ccfb2. 00000020:00000010:1.0:1775764548.116273:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000d7502ed5. 00000020:00000010:1.0:1775764548.116344:0:6179:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cf9df0d9. 00000020:00000010:1.0:1775764548.116350:0:6179:0:(tgt_main.c:769:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000e840a280. 00000008:00000010:1.0:1775764548.116355:0:6179:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000ed0ce56e. 00020000:00000010:1.0:1775764548.116359:0:6179:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000009a72f5f6. 00008000:00000010:1.0:1775764548.116362:0:6179:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 00000000eefcea5b. 00000080:00000010:1.0:1775764548.116365:0:6179:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000b81dcbdc. 00000020:00000010:1.0:1775764548.116369:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000f777ccb4. 00000020:00000010:1.0:1775764548.116377:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000048b90b88. 00000020:00000010:1.0:1775764548.116380:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 39 at 000000008491c33c. 00000020:00000010:1.0:1775764548.116391:0:6179:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 000000008491c33c. 00000020:00000010:1.0:1775764548.116395:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at 00000000cf9df0d9. 00000020:00000010:1.0:1775764548.116404:0:6179:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b5040633. 00000020:00000010:1.0:1775764548.116408:0:6179:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000007f8aeb20. 00000020:00000010:1.0:1775764548.116412:0:6179:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000cff16a2e. 00000008:00000010:1.0:1775764548.116416:0:6179:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000003049a7b2. 00020000:00000010:1.0:1775764548.116419:0:6179:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000009e0a8aa. 00008000:00000010:1.0:1775764548.116422:0:6179:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000cac43077. 00000080:00000010:1.0:1775764548.116426:0:6179:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000016ac4737. 00000080:00000010:1.0:1775764548.116429:0:6179:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000ac0ccfb2. 00020000:00000010:1.0:1775764548.116450:0:6179:0:(lov_obd.c:495:lov_add_target()) kmalloced '(tgt)': 368 at 000000000cbd10eb. 00000020:00000010:1.0:1775764548.116458:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000007f8aeb20. 00000020:00000010:1.0:1775764548.116460:0:6179:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000cff16a2e. 00000008:00000010:1.0:1775764548.116462:0:6179:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000003049a7b2. 00020000:00000010:1.0:1775764548.116464:0:6179:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 0000000009e0a8aa. 00008000:00000010:1.0:1775764548.116466:0:6179:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000cac43077. 00000080:00000010:1.0:1775764548.116468:0:6179:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000016ac4737. 00000080:00000010:1.0:1775764548.116469:0:6179:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000ac0ccfb2. 00000020:00000010:1.0:1775764548.116471:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b5040633. 00000020:00000010:1.0:1775764548.116473:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 120 at 00000000cf9df0d9. 00000020:00000010:1.0:1775764548.116475:0:6179:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 34 at 000000008491c33c. 00000020:00000010:1.0:1775764548.116480:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000003296d428. 00000020:00000010:1.0:1775764548.116482:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000003296d428. 00000020:00000010:1.0:1775764548.116504:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000003296d428. 00000020:00000010:1.0:1775764548.116506:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000003296d428. 00000020:00000010:1.0:1775764548.116509:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 56 at 000000008491c33c. 00000020:00000010:1.0:1775764548.116512:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 56 at 000000008491c33c. 00000100:00000010:1.0:1775764548.116522:0:6179:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000069b21deb. 02000000:00000010:1.0:1775764548.116600:0:6179:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000b5040633. 00000100:00000010:1.0:1775764548.116623:0:6179:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000b08a2b85. 02000000:00000010:1.0:1775764548.116655:0:6179:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 000000009f2d3498. 00000400:00000200:1.0:1775764548.116668:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:1.0:1775764548.116672:0:6179:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e8be07c9. 00000400:00000010:1.0:1775764548.117886:0:6179:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 000000003864c88d. 00000100:00000200:1.0:1775764548.117900:0:6179:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1862023861240832, portal 25 00000400:00000010:1.0:1775764548.117935:0:6179:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000852ce55c. 00000100:00000200:1.0:1775764548.117940:0:6179:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1862023861240832, offset 0 00000400:00000200:1.0:1775764548.117950:0:6179:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:1.0:1775764548.117961:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.117969:0:6179:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:1.0:1775764548.117974:0:6179:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775764548.117978:0:6179:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:1.0:1775764548.117988:0:6179:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775764548.117991:0:6179:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:1.0:1775764548.117994:0:6179:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [11:11:256:4294967295] 192.168.202.104@tcp LPNI seq info [11:11:8:4294967295] 00000400:00000200:1.0:1775764548.118000:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.118010:0:6179:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:1.0:1775764548.118016:0:6179:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:1.0:1775764548.118021:0:6179:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003296d428. 00000800:00000200:1.0:1775764548.118028:0:6179:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:1.0:1775764548.118036:0:6179:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:1.0:1775764548.118041:0:6179:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000003296d428 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:1.0:1775764548.118055:0:6179:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:1.0:1775764548.118160:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.118163:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003296d428. 00000400:00000200:1.0:1775764548.118167:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.118172:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.118178:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000852ce55c 00000400:00000010:1.0:1775764548.118180:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000852ce55c. 00000100:00000200:1.0:1775764548.118188:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000069b21deb x1862023861240832/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.118211:0:6179:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000cff16a2e. 00000100:00000010:1.0:1775764548.118225:0:6179:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000dee0a3ca. 00000400:00000200:1.0:1775764548.121451:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.121463:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.121474:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.121480:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.121527:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.121532:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 8472 into portal 25 MB=0x69d8036d00400 00000400:00000200:1.0:1775764548.121850:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.121867:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 8472/8472 into md 0x269 [4] + 224 00000400:00000200:1.0:1775764548.121896:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.121920:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.121935:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000069b21deb x1862023861240832/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.121997:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000dee0a3ca. 00000100:00000010:0.0:1775764548.122012:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000cff16a2e. 00000400:00000010:2.0:1775764548.122080:0:6179:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000e8be07c9. 00000400:00000200:2.0:1775764548.122087:0:6179:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000003864c88d 00000400:00000010:2.0:1775764548.122090:0:6179:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 000000003864c88d. 00000100:00000200:2.0:1775764548.122096:0:6179:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000069b21deb x1862023861240832/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.122118:0:6179:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000069b21deb x1862023861240832/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764548.122157:0:6179:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000b08a2b85. 02000000:00000010:2.0:1775764548.122174:0:6179:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 000000009f2d3498. 02000000:00000010:2.0:1775764548.122191:0:6179:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000b5040633. 00000100:00000010:2.0:1775764548.122197:0:6179:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000069b21deb. 00000020:00000010:2.0:1775764548.122219:0:6179:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000cc260a03. 00000020:00000010:2.0:1775764548.122225:0:6179:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000cc260a03. 00000040:00000010:2.0:1775764548.122230:0:6179:0:(llog.c:894:llog_process_thread()) kvfree 'buf': 8192 at 000000000d38ef68. 00000020:00000010:2.0:1775764548.122239:0:6179:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at 000000007582954d. 00000020:00000010:2.0:1775764548.122251:0:6179:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000008947adc5. 00000040:00000010:2.0:1775764548.122256:0:6179:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 000000009bbda9a2. 00000020:00000010:2.0:1775764548.122261:0:6179:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000051531c9. 00000020:00000010:2.0:1775764548.122268:0:6179:0:(tgt_main.c:740:tgt_key_fini()) slab-freed '(info)': 1288 at 00000000acc5946a. 00000020:00000010:2.0:1775764548.122274:0:6179:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 0000000053d89429. 80000000:00000010:2.0:1775764548.122279:0:6179:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 00000000f1a43e15. 40000000:00000010:2.0:1775764548.122284:0:6179:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at 0000000047a74c5e. 00100000:00000010:2.0:1775764548.122290:0:6179:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at 000000009edb78ef. 00040000:00000010:2.0:1775764548.122296:0:6179:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 0000000017a02ccd. 00040000:00000010:2.0:1775764548.122302:0:6179:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at 00000000eed95878. 00040000:00000010:2.0:1775764548.122306:0:6179:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 0000000061f83f4b. 00000004:00000010:2.0:1775764548.122309:0:6179:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at 00000000044f57bb. 00000004:00000010:2.0:1775764548.122315:0:6179:0:(mdt_handler.c:8411:mdt_key_fini()) kfreed 'info': 2568 at 00000000b6fdc258. 00000004:00000010:2.0:1775764548.122322:0:6179:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at 000000004de738fa. 00002000:00000010:2.0:1775764548.122326:0:6179:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at 000000007ca6751b. 00000004:00000010:2.0:1775764548.122331:0:6179:0:(osp_dev.c:1823:osp_key_fini()) kfreed 'info': 616 at 00000000bc041687. 00000020:00000010:2.0:1775764548.122333:0:6179:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000002884ac61. 00000040:00000010:3.0:1775764548.122533:0:6160:0:(llog.c:989:llog_process_or_fork()) kfreed 'lpi': 96 at 000000004f52e8d5. 00000040:00000010:3.0:1775764548.122544:0:6160:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at 00000000702bfb51. 00000040:00000010:3.0:1775764548.122548:0:6160:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.122554:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000dc578cdd. 00000040:00000010:3.0:1775764548.122557:0:6160:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 000000001e426763. 00000020:00000010:3.0:1775764548.122561:0:6160:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000f602d917. 00000020:00000010:3.0:1775764548.122567:0:6160:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000c7915c02. 80000000:00000010:3.0:1775764548.122569:0:6160:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 000000002a4f10c1. 00040000:00000010:3.0:1775764548.122572:0:6160:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000a95ec3e3. 20000000:00000010:3.0:1775764548.122576:0:6160:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at 0000000031e063b1. 00000020:00000010:3.0:1775764548.122579:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000000a45a8e4. 10000000:00000010:3.0:1775764548.122582:0:6160:0:(mgc_request.c:1748:mgc_process_cfg_log()) kfreed 'env': 56 at 000000000e92995b. 00000100:00000010:3.0:1775764548.122614:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000944a1a9f. 02000000:00000010:3.0:1775764548.122631:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 0000000031e063b1. 00010000:00000010:3.0:1775764548.122649:0:6160:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000421a0bea. 00010000:00000010:3.0:1775764548.122659:0:6160:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 000000009ecf5f76. 00000100:00000010:3.0:1775764548.122694:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bbed3f13. 02000000:00000010:3.0:1775764548.122718:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000aaa76d25. 00000400:00000200:3.0:1775764548.122726:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764548.122729:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b7689a63. 00000400:00000010:3.0:1775764548.122734:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d0e127ae. 00000100:00000200:3.0:1775764548.122742:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861240960, portal 25 00000400:00000010:3.0:1775764548.122748:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d9fc6616. 00000100:00000200:3.0:1775764548.122752:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1862023861240960, offset 0 00000400:00000200:3.0:1775764548.122760:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764548.122768:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.122778:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764548.122786:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775764548.122790:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:3.0:1775764548.122806:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775764548.122816:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.122820:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [12:12:256:4294967295] 192.168.202.104@tcp LPNI seq info [12:12:8:4294967295] 00000400:00000200:3.0:1775764548.122824:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.122835:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764548.122841:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764548.122845:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006e5f5541. 00000800:00000200:3.0:1775764548.122853:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:3.0:1775764548.122870:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764548.122874:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000006e5f5541 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1775764548.122893:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:3.0:1775764548.122899:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000d8e4a5e3. 00000100:00000010:3.0:1775764548.122909:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002a4f10c1. 00000800:00000200:1.0:1775764548.123002:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.123006:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006e5f5541. 00000400:00000200:1.0:1775764548.123012:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.123016:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.123020:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d9fc6616 00000400:00000010:1.0:1775764548.123021:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000d9fc6616. 00000100:00000200:1.0:1775764548.123027:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000944a1a9f x1862023861240960/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.129809:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002a4f10c1. 00000100:00000010:0.0:1775764548.131403:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000d8e4a5e3. 00000400:00000200:0.0:1775764548.132427:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.132438:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.132448:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.132453:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.132460:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.132464:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 344 into portal 25 MB=0x69d8036d00480 00000400:00000200:0.0:1775764548.132471:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.132478:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 344/344 into md 0x279 [1] + 224 00000400:00000200:0.0:1775764548.132548:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.132554:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.132573:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000944a1a9f x1862023861240960/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775764548.132689:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b7689a63. 00000400:00000200:3.0:1775764548.132695:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d0e127ae 00000400:00000010:3.0:1775764548.132698:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000d0e127ae. 00000100:00000200:3.0:1775764548.132703:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000944a1a9f x1862023861240960/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775764548.132711:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000944a1a9f x1862023861240960/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775764548.132739:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bbed3f13. 02000000:00000010:3.0:1775764548.132766:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000aaa76d25. 02000000:00000010:3.0:1775764548.132773:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 0000000031e063b1. 00000100:00000010:3.0:1775764548.132776:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000944a1a9f. 10000000:00000010:3.0:1775764548.132787:0:6160:0:(mgc_request.c:1551:mgc_process_recover_log()) kmalloced '(pages)': 2048 at 000000009762d1e8. 00000100:00000010:3.0:1775764548.133298:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000944a1a9f. 02000000:00000010:3.0:1775764548.133315:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 0000000031e063b1. 00000100:00000010:3.0:1775764548.133326:0:6160:0:(client.c:169:ptlrpc_new_bulk()) kmalloced '(desc)': 984 at 00000000aaa76d25. 00000100:00000010:3.0:1775764548.133330:0:6160:0:(client.c:174:ptlrpc_new_bulk()) kmalloced '(desc->bd_vec)': 4096 at 00000000ae7d466f. 00000100:00000010:3.0:1775764548.133348:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bbed3f13. 00000400:00000200:3.0:1775764548.133372:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764548.133380:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b7689a63. 00000400:00000010:3.0:1775764548.133387:0:6160:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 4216 at 00000000053fbe3d. 00000100:00000200:3.0:1775764548.133395:0:6160:0:(niobuf.c:453:ptlrpc_register_bulk()) Setup 1 bulk put-sink buffers: 256 pages 1048576 bytes, mbits x0x69d8036d00500-0x69d8036d00500, portal 33 02000000:00000010:3.0:1775764548.133402:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a95ec3e3. 00000400:00000200:3.0:1775764548.133405:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764548.133407:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007412d5a2. 00000400:00000010:3.0:1775764548.133409:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d0e127ae. 00000100:00000200:3.0:1775764548.133412:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1862023861241088, portal 25 00000400:00000010:3.0:1775764548.133417:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000176af6a5. 00000100:00000200:3.0:1775764548.133420:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 304 bytes to portal 26, xid 1862023861241088, offset 0 00000400:00000200:3.0:1775764548.133428:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764548.133436:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.133443:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764548.133448:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775764548.133451:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:3.0:1775764548.133455:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775764548.133457:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.133461:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [13:13:256:4294967295] 192.168.202.104@tcp LPNI seq info [13:13:8:4294967295] 00000400:00000200:3.0:1775764548.133464:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.133472:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764548.133477:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 304 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764548.133520:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000070596667. 00000800:00000200:3.0:1775764548.133528:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:3.0:1775764548.133535:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764548.133539:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000070596667 type 1, nob 400 niov 1 nkiov 1 00000400:00000200:3.0:1775764548.133560:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:3.0:1775764548.133564:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000004f52e8d5. 00000100:00000010:3.0:1775764548.133580:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000002a4f10c1. 00000800:00000200:1.0:1775764548.134297:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.134302:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000070596667. 00000400:00000200:1.0:1775764548.134310:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.134317:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.134322:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000176af6a5 00000400:00000010:1.0:1775764548.134324:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000176af6a5. 00000100:00000200:1.0:1775764548.134332:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000944a1a9f x1862023861241088/t0(0) o256->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 304/240 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775764548.135930:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.135940:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.135950:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.135955:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.135961:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.135965:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 176 into portal 33 MB=0x69d8036d00500 00000400:00000200:0.0:1775764548.135972:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.135980:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 21 from 12345-192.168.202.104@tcp of length 176/176 into md 0x281 [256] + 0 00000400:00000010:0.0:1775764548.136008:0:2394:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b7689a63. 00000400:00000200:0.0:1775764548.136014:0:2394:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000053fbe3d 00000400:00000200:0.0:1775764548.136033:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.136038:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.136043:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000053fbe3d 00000400:00000010:0.0:1775764548.136045:0:2394:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 4216 at 00000000053fbe3d. 00000100:00000200:0.0:1775764548.136052:0:2394:0:(events.c:192:client_bulk_callback()) event type 2, status 0, req 00000000944a1a9f desc 00000000aaa76d25 mbits 1862023861241088 00000400:00000200:0.0:1775764548.136081:0:2394:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:0.0:1775764548.136086:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.136093:0:2394:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source Specified: 192.168.202.4@tcp to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764548.136100:0:2394:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [14:14:256:4294967295] 192.168.202.104@tcp LPNI seq info [14:14:8:4294967295] 00000400:00000200:0.0:1775764548.136104:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.136113:0:2394:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:192.168.202.4@tcp) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) 192.168.202.104@tcp : ACK try# 0 00000800:00000200:0.0:1775764548.136118:0:2394:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 256 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764548.136127:0:2394:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 4328 at 000000002c71c866. 00000800:00000200:0.0:1775764548.136135:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:0.0:1775764548.136142:0:2394:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764548.136147:0:2394:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000002c71c866 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1775764548.136250:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.136254:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 4328 at 000000002c71c866. 00000400:00000200:0.0:1775764548.136259:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.136263:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: ACK: OK 00000400:00000200:0.0:1775764548.138848:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.138868:0:2394:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:0.0:1775764548.138877:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.138882:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.138888:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.138892:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 240 into portal 25 MB=0x69d8036d00500 00000400:00000200:0.0:1775764548.138898:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.138904:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 240/240 into md 0x289 [1] + 224 00000400:00000200:0.0:1775764548.138935:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.138940:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:0.0:1775764548.138954:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000944a1a9f x1862023861241088/t0(0) o256->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 304/240 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775764548.139102:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000007412d5a2. 00000400:00000200:3.0:1775764548.139107:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d0e127ae 00000400:00000010:3.0:1775764548.139109:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000d0e127ae. 00000100:00000200:3.0:1775764548.139121:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000944a1a9f x1862023861241088/t0(0) o256->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 304/240 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775764548.139130:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000944a1a9f x1862023861241088/t0(0) o256->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 304/240 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775764548.139162:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bbed3f13. 10000000:00000010:3.0:1775764548.139175:0:6160:0:(mgc_request.c:1334:mgc_apply_recover_logs()) kmalloced '(nidlist)': 640 at 00000000edebe51d. 10000000:00000010:3.0:1775764548.139180:0:6160:0:(mgc_request.c:1338:mgc_apply_recover_logs()) kmalloced '(buf)': 4096 at 00000000f8c318a3. 10000000:00000200:3.0:1775764548.139185:0:6160:0:(mgc_request.c:1239:mgc_build_nidlist()) use 1 from 1 NIDs 00000100:00000200:3.0:1775764548.139196:0:6160:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.202.104@tcp with 192.168.202.104@tcp 00000100:00000200:3.0:1775764548.139199:0:6160:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.202.104@tcp' 00000400:00000200:3.0:1775764548.139201:0:6160:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1775764548.139205:0:6160:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.139209:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.139212:0:6160:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1775764548.139215:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.139220:0:6160:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1775764548.139222:0:6160:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:3.0:1775764548.139225:0:6160:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 10000000:00000010:3.0:1775764548.139230:0:6160:0:(mgc_request.c:1492:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000ffe4b4ea. 00000020:00000010:3.0:1775764548.139240:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c7915c02. 00000020:00000010:3.0:1775764548.139245:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.139250:0:6160:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.139257:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.139262:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.139266:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.139273:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.139277:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000000fa84934. 00000100:00000010:1.0:1775764548.139359:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000002a4f10c1. 00000020:00000010:3.0:1775764548.139405:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.139410:0:6160:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.139413:0:6160:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.139416:0:6160:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.139419:0:6160:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.139428:0:6160:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.139431:0:6160:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000000fa84934. 00000020:00000010:3.0:1775764548.139433:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c7915c02. 10000000:00000010:3.0:1775764548.139438:0:6160:0:(mgc_request.c:1504:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000ffe4b4ea. 10000000:00000200:3.0:1775764548.139441:0:6160:0:(mgc_request.c:1239:mgc_build_nidlist()) use 1 from 1 NIDs 00000100:00000200:3.0:1775764548.139447:0:6160:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.202.104@tcp with 192.168.202.104@tcp 00000100:00000200:3.0:1775764548.139449:0:6160:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.202.104@tcp' 00000400:00000200:3.0:1775764548.139450:0:6160:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1775764548.139453:0:6160:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.139456:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.139458:0:6160:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1775764548.139460:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.139462:0:6160:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1775764548.139464:0:6160:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:3.0:1775764548.139466:0:6160:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 10000000:00000010:3.0:1775764548.139469:0:6160:0:(mgc_request.c:1492:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000ffe4b4ea. 00000020:00000010:3.0:1775764548.139473:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c7915c02. 00000020:00000010:3.0:1775764548.139475:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.139477:0:6160:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.139479:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.139481:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.139516:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.139519:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.139521:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000000fa84934. 00000100:00000010:0.0:1775764548.139588:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000004f52e8d5. 00000020:00000010:3.0:1775764548.140514:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.140520:0:6160:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.140523:0:6160:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.140526:0:6160:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.140529:0:6160:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.140532:0:6160:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.140534:0:6160:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000000fa84934. 00000020:00000010:3.0:1775764548.140536:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c7915c02. 10000000:00000010:3.0:1775764548.140540:0:6160:0:(mgc_request.c:1504:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000ffe4b4ea. 10000000:00000200:3.0:1775764548.140551:0:6160:0:(mgc_request.c:1239:mgc_build_nidlist()) use 1 from 1 NIDs 00000100:00000200:3.0:1775764548.140561:0:6160:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.202.104@tcp with 192.168.202.104@tcp 00000100:00000200:3.0:1775764548.140563:0:6160:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.202.104@tcp' 00000400:00000200:3.0:1775764548.140565:0:6160:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1775764548.140569:0:6160:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.140574:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.140577:0:6160:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1775764548.140580:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.140583:0:6160:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1775764548.140585:0:6160:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:3.0:1775764548.140588:0:6160:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 10000000:00000010:3.0:1775764548.140591:0:6160:0:(mgc_request.c:1492:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000ffe4b4ea. 00000020:00000010:3.0:1775764548.140598:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c7915c02. 00000020:00000010:3.0:1775764548.140601:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.140604:0:6160:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.140607:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.140610:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.140612:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.140616:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.140618:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000000fa84934. 00000020:00000010:3.0:1775764548.140649:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.140652:0:6160:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.140654:0:6160:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.140655:0:6160:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.140657:0:6160:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.140659:0:6160:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.140672:0:6160:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000000fa84934. 00000020:00000010:3.0:1775764548.140675:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c7915c02. 10000000:00000010:3.0:1775764548.140678:0:6160:0:(mgc_request.c:1504:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000ffe4b4ea. 10000000:00000200:3.0:1775764548.140680:0:6160:0:(mgc_request.c:1239:mgc_build_nidlist()) use 1 from 1 NIDs 00000100:00000200:3.0:1775764548.140686:0:6160:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.202.104@tcp with 192.168.202.104@tcp 00000100:00000200:3.0:1775764548.140687:0:6160:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.202.104@tcp' 00000400:00000200:3.0:1775764548.140688:0:6160:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1775764548.140691:0:6160:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.140698:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.140700:0:6160:0:(peer.c:1746:lnet_peer_add()) peer 192.168.202.104@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1775764548.140703:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.140705:0:6160:0:(peer.c:2776:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1775764548.140708:0:6160:0:(peer.c:2785:lnet_discover_peer_locked()) peer 192.168.202.104@tcp NID 192.168.202.104@tcp: 0. pending discovery 00000400:00000200:3.0:1775764548.140709:0:6160:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.202.104@tcp 10000000:00000010:3.0:1775764548.140712:0:6160:0:(mgc_request.c:1492:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000ffe4b4ea. 00000020:00000010:3.0:1775764548.140717:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c7915c02. 00000020:00000010:3.0:1775764548.140719:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.140722:0:6160:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.140724:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.140726:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.140729:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.140731:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.140733:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000000fa84934. 00000020:00000010:3.0:1775764548.140766:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000a3f385fa. 00000020:00000010:3.0:1775764548.140769:0:6160:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000596fe25f. 00000008:00000010:3.0:1775764548.140771:0:6160:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000065792a79. 00020000:00000010:3.0:1775764548.140773:0:6160:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 00000000196a4339. 00008000:00000010:3.0:1775764548.140775:0:6160:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 000000005937f1e8. 00000080:00000010:3.0:1775764548.140777:0:6160:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000062688834. 00000080:00000010:3.0:1775764548.140779:0:6160:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000000fa84934. 00000020:00000010:3.0:1775764548.140781:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c7915c02. 10000000:00000010:3.0:1775764548.140784:0:6160:0:(mgc_request.c:1504:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000ffe4b4ea. 10000000:00000010:3.0:1775764548.140786:0:6160:0:(mgc_request.c:1511:mgc_apply_recover_logs()) kfreed 'buf': 4096 at 00000000f8c318a3. 10000000:00000010:3.0:1775764548.140788:0:6160:0:(mgc_request.c:1513:mgc_apply_recover_logs()) kfreed 'nidlist': 640 at 00000000edebe51d. 02000000:00000010:3.0:1775764548.140806:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a95ec3e3. 00000100:00000010:3.0:1775764548.140830:0:6160:0:(client.c:333:ptlrpc_free_bulk()) kvfree 'desc->bd_vec': 4096 at 00000000ae7d466f. 00000100:00000010:3.0:1775764548.140834:0:6160:0:(client.c:334:ptlrpc_free_bulk()) kfreed 'desc': 984 at 00000000aaa76d25. 02000000:00000010:3.0:1775764548.140837:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 0000000031e063b1. 00000100:00000010:3.0:1775764548.140841:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000944a1a9f. 10000000:00000010:3.0:1775764548.141360:0:6160:0:(mgc_request.c:1669:mgc_process_recover_log()) kvfree 'pages': 2048 at 000000009762d1e8. 00000100:00000010:3.0:1775764548.141389:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000944a1a9f. 02000000:00000010:3.0:1775764548.141410:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 0000000031e063b1. 00010000:00000010:3.0:1775764548.141424:0:6160:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000f89ea212. 00010000:00000010:3.0:1775764548.141438:0:6160:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000bd29a72e. 00000100:00000010:3.0:1775764548.141458:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bbed3f13. 02000000:00000010:3.0:1775764548.141476:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f6bf30dc. 00000400:00000200:3.0:1775764548.141481:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764548.141512:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007412d5a2. 00000400:00000010:3.0:1775764548.141517:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d0e127ae. 00000100:00000200:3.0:1775764548.141524:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861241216, portal 25 00000400:00000010:3.0:1775764548.141528:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000005770723. 00000100:00000200:3.0:1775764548.141531:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1862023861241216, offset 0 00000400:00000200:3.0:1775764548.141540:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764548.141548:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.141553:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764548.141558:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775764548.141563:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:3.0:1775764548.141567:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775764548.141569:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.141573:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [15:15:256:4294967295] 192.168.202.104@tcp LPNI seq info [15:15:8:4294967295] 00000400:00000200:3.0:1775764548.141578:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.141588:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764548.141593:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764548.141599:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000070596667. 00000800:00000200:3.0:1775764548.141606:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:3.0:1775764548.141614:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764548.141618:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000070596667 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1775764548.141642:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:3.0:1775764548.141646:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000596fe25f. 00000100:00000010:3.0:1775764548.141659:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000dccb75fe. 00000800:00000200:1.0:1775764548.141939:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.141942:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000070596667. 00000400:00000200:1.0:1775764548.141948:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.141954:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.141959:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000005770723 00000400:00000010:1.0:1775764548.141962:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000005770723. 00000100:00000200:1.0:1775764548.141970:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000944a1a9f x1862023861241216/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.146241:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000dccb75fe. 00000400:00000200:1.0:1775764548.150531:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.150540:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.150550:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.150555:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.150560:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.150564:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 344 into portal 25 MB=0x69d8036d00580 00000400:00000200:1.0:1775764548.150571:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.150576:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 344/344 into md 0x291 [1] + 224 00000400:00000200:1.0:1775764548.150596:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.150602:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.150621:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000944a1a9f x1862023861241216/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775764548.150895:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000007412d5a2. 00000400:00000200:3.0:1775764548.150902:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d0e127ae 00000400:00000010:3.0:1775764548.150922:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000d0e127ae. 00000100:00000200:3.0:1775764548.150929:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000944a1a9f x1862023861241216/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775764548.150938:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000944a1a9f x1862023861241216/t0(0) o101->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 328/344 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775764548.150974:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bbed3f13. 02000000:00000010:3.0:1775764548.151009:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f6bf30dc. 02000000:00000010:3.0:1775764548.151015:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 0000000031e063b1. 00000100:00000010:3.0:1775764548.151021:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000944a1a9f. 10000000:00000010:3.0:1775764548.151035:0:6160:0:(mgc_request.c:1700:mgc_process_cfg_log()) kmalloced '(env)': 56 at 0000000015f8035e. 00000020:00000010:3.0:1775764548.151040:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000031e063b1. 00000020:00000010:3.0:1775764548.151048:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a95ec3e3. 00000040:00000010:3.0:1775764548.151052:0:6160:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000c7915c02. 00000020:00000010:3.0:1775764548.151058:0:6160:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000009762d1e8. 00000020:00000010:3.0:1775764548.151064:0:6160:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000a3f385fa. 80000000:00000010:3.0:1775764548.151072:0:6160:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000dccb75fe. 00040000:00000010:3.0:1775764548.151079:0:6160:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000b5f5bca5. 20000000:00000010:3.0:1775764548.151084:0:6160:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 00000000a7135789. 00000040:00000010:3.0:1775764548.151093:0:6160:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 000000000a45a8e4. 00000100:00000010:3.0:1775764548.151100:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000944a1a9f. 02000000:00000010:3.0:1775764548.151119:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000dc578cdd. 00000100:00000010:3.0:1775764548.151134:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bbed3f13. 02000000:00000010:3.0:1775764548.151157:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000057324ad. 00000400:00000200:3.0:1775764548.151164:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764548.151166:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007412d5a2. 00000400:00000010:3.0:1775764548.151171:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d0e127ae. 00000100:00000200:3.0:1775764548.151180:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1862023861241344, portal 25 00000400:00000010:3.0:1775764548.151185:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ca0c22b3. 00000100:00000200:3.0:1775764548.151188:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 504 bytes to portal 26, xid 1862023861241344, offset 0 00000400:00000200:3.0:1775764548.151196:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764548.151202:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.151211:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764548.151223:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775764548.151230:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:3.0:1775764548.151234:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775764548.151237:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:3.0:1775764548.151242:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [16:16:256:4294967295] 192.168.202.104@tcp LPNI seq info [16:16:8:4294967295] 00000400:00000200:3.0:1775764548.151247:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764548.151258:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764548.151264:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 504 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764548.151270:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006e5f5541. 00000800:00000200:3.0:1775764548.151278:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:3.0:1775764548.151289:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764548.151293:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000006e5f5541 type 1, nob 600 niov 1 nkiov 1 00000400:00000200:3.0:1775764548.151317:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:3.0:1775764548.151333:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000a9967bd0. 00000800:00000200:1.0:1775764548.153233:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.153244:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006e5f5541. 00000400:00000200:1.0:1775764548.153257:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.153267:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.153275:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000ca0c22b3 00000400:00000010:1.0:1775764548.153278:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000ca0c22b3. 00000100:00000200:1.0:1775764548.153291:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000944a1a9f x1862023861241344/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 504/272 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775764548.158301:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000596fe25f. 00000400:00000200:1.0:1775764548.158973:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.158982:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.158992:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.158998:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.159003:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.159007:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 272 into portal 25 MB=0x69d8036d00600 00000400:00000200:1.0:1775764548.159012:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.159018:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 272/272 into md 0x299 [1] + 224 00000400:00000200:1.0:1775764548.159039:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.159044:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.159058:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000944a1a9f x1862023861241344/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 504/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.162073:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000a9967bd0. 00000400:00000010:2.0:1775764548.167737:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000007412d5a2. 00000400:00000200:2.0:1775764548.167758:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d0e127ae 00000400:00000010:2.0:1775764548.167761:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000d0e127ae. 00000100:00000200:2.0:1775764548.167784:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000944a1a9f x1862023861241344/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 504/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.170882:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000944a1a9f x1862023861241344/t0(0) o501->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 504/272 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764548.170920:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bbed3f13. 02000000:00000010:2.0:1775764548.170938:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000057324ad. 02000000:00000010:2.0:1775764548.170945:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000dc578cdd. 00000100:00000010:2.0:1775764548.170951:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000944a1a9f. 00000040:00000010:2.0:1775764548.170965:0:6160:0:(llog.c:415:llog_init_handle()) kmalloced '(llh)': 8192 at 00000000f05f495f. 00000100:00000010:2.0:1775764548.170972:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000277fa9c1. 02000000:00000010:2.0:1775764548.170987:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000009559b35c. 00000100:00000010:2.0:1775764548.171001:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000f8e99454. 02000000:00000010:2.0:1775764548.172868:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 000000005ef9ca6b. 00000400:00000200:2.0:1775764548.172894:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764548.172901:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006f22505c. 00000400:00000010:2.0:1775764548.172910:0:6160:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 0000000086a2ae3f. 00000100:00000200:2.0:1775764548.172920:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1862023861241472, portal 25 00000400:00000010:2.0:1775764548.172931:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000da1a8a2e. 00000100:00000200:2.0:1775764548.172935:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1862023861241472, offset 0 00000400:00000200:2.0:1775764548.172948:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.172958:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.172967:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764548.172973:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764548.172978:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764548.172982:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764548.172985:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764548.172988:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [17:17:256:4294967295] 192.168.202.104@tcp LPNI seq info [17:17:8:4294967295] 00000400:00000200:2.0:1775764548.172993:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.173003:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764548.173010:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764548.173016:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000cc260a03. 00000800:00000200:2.0:1775764548.173024:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764548.173041:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764548.173045:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000cc260a03 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1775764548.173077:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:2.0:1775764548.173083:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000048f66367. 00000100:00000010:2.0:1775764548.173098:0:6160:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 0000000048e2fa4d. 00000800:00000200:1.0:1775764548.173428:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.173434:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000cc260a03. 00000400:00000200:1.0:1775764548.173440:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.173446:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.173452:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000da1a8a2e 00000400:00000010:1.0:1775764548.173454:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000da1a8a2e. 00000100:00000200:1.0:1775764548.173463:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000277fa9c1 x1862023861241472/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.176583:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000048e2fa4d. 00000400:00000200:1.0:1775764548.187802:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.187813:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.187826:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.187831:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.187838:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.187843:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 8416 into portal 25 MB=0x69d8036d00680 00000400:00000200:1.0:1775764548.187848:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.187854:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 8416/8416 into md 0x2a1 [4] + 224 00000400:00000200:1.0:1775764548.187889:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.187893:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.187907:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000277fa9c1 x1862023861241472/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775764548.187964:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000006f22505c. 00000400:00000200:1.0:1775764548.187968:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000086a2ae3f 00000400:00000010:1.0:1775764548.187971:0:6160:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 0000000086a2ae3f. 00000100:00000200:1.0:1775764548.187975:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000277fa9c1 x1862023861241472/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775764548.187983:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000277fa9c1 x1862023861241472/t0(0) o503->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8416 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.188026:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000f8e99454. 02000000:00000010:1.0:1775764548.188041:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 000000005ef9ca6b. 02000000:00000010:1.0:1775764548.188056:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000009559b35c. 00000100:00000010:1.0:1775764548.188062:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000277fa9c1. 00000040:00000010:1.0:1775764548.188077:0:6160:0:(llog.c:948:llog_process_or_fork()) kmalloced '(lpi)': 96 at 00000000dee0a3ca. 00000020:00000010:1.0:1775764548.188582:0:6188:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at 000000004550cadb. 00000020:00000010:1.0:1775764548.188594:0:6188:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at 000000004550cadb. 00000020:00000010:2.0:1775764548.196713:0:6190:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008ce176f5. 00000020:00000010:2.0:1775764548.196730:0:6190:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000351a47a0. 00000040:00000010:2.0:1775764548.196734:0:6190:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000a7fda8c4. 00000020:00000010:2.0:1775764548.196739:0:6190:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000068f9c321. 00000020:00000010:2.0:1775764548.196745:0:6190:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000b2a1933b. 00000020:00000010:2.0:1775764548.196750:0:6190:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000e7657df0. 80000000:00000010:2.0:1775764548.196754:0:6190:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000009b833da6. 40000000:00000010:2.0:1775764548.196759:0:6190:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000003ab4608d. 00100000:00000010:2.0:1775764548.196765:0:6190:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000002569afed. 00040000:00000010:2.0:1775764548.196769:0:6190:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000d6135371. 00040000:00000010:2.0:1775764548.196774:0:6190:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000c0d73163. 00040000:00000010:2.0:1775764548.196777:0:6190:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000004ef70566. 00000004:00000010:2.0:1775764548.196781:0:6190:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000069d9d49b. 00000004:00000010:2.0:1775764548.196787:0:6190:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at 0000000083f226da. 00000004:00000010:2.0:1775764548.196962:0:6190:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000009efdc4be. 00002000:00000010:2.0:1775764548.196970:0:6190:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000d31ed978. 00000004:00000010:2.0:1775764548.196974:0:6190:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at 00000000d29f1a24. 00000020:00000010:2.0:1775764548.196978:0:6190:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000053bdb9b2. 00000040:00000010:2.0:1775764548.196988:0:6190:0:(llog.c:573:llog_process_thread()) kmalloced '(buf)': 8192 at 00000000702bfb51. 00000100:00000010:2.0:1775764548.196999:0:6190:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000c1e17336. 02000000:00000010:2.0:1775764548.197016:0:6190:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000003884c2aa. 00000100:00000010:2.0:1775764548.197036:0:6190:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000f091afaf. 02000000:00000010:2.0:1775764548.197080:0:6190:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 00000000cc4821c3. 00000400:00000200:2.0:1775764548.197092:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764548.197097:0:6190:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000252feeb6. 00000400:00000010:2.0:1775764548.197104:0:6190:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 00000000c53a1658. 00000100:00000200:2.0:1775764548.197112:0:6190:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1862023861241600, portal 25 00000400:00000010:2.0:1775764548.197125:0:6190:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c898c0d8. 00000100:00000200:2.0:1775764548.197128:0:6190:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1862023861241600, offset 0 00000400:00000200:2.0:1775764548.197140:0:6190:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.197151:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.197162:0:6190:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764548.197168:0:6190:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764548.197174:0:6190:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764548.197179:0:6190:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764548.197182:0:6190:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764548.197186:0:6190:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [18:18:256:4294967295] 192.168.202.104@tcp LPNI seq info [18:18:8:4294967295] 00000400:00000200:2.0:1775764548.197192:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.197204:0:6190:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764548.197211:0:6190:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764548.197216:0:6190:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000cc260a03. 00000800:00000200:2.0:1775764548.197223:0:6190:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001f4dd10c] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764548.197232:0:6190:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764548.197237:0:6190:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000cc260a03 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1775764548.197260:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:2.0:1775764548.197275:0:6190:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000254b12e0. 00000020:00000010:3.0:1775764548.197965:0:6187:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at 00000000d8ad15e1. 00000020:00000010:3.0:1775764548.197976:0:6187:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at 00000000d8ad15e1. 00000800:00000200:1.0:1775764548.199641:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.199650:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000cc260a03. 00000400:00000200:1.0:1775764548.199658:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.200113:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.200123:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c898c0d8 00000400:00000010:1.0:1775764548.200126:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c898c0d8. 00000100:00000200:1.0:1775764548.200142:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000c1e17336 x1862023861241600/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.200193:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000254b12e0. 00000100:00000010:0.0:1775764548.209570:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 0000000048f66367. 00000400:00000200:1.0:1775764548.216148:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.216160:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.216170:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.216175:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.216182:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.216186:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 8472 into portal 25 MB=0x69d8036d00700 00000400:00000200:1.0:1775764548.216192:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.216198:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 8472/8472 into md 0x2a9 [4] + 224 00000400:00000200:1.0:1775764548.216227:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.216233:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.216249:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000c1e17336 x1862023861241600/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775764548.224287:0:6193:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at 000000006aaf5fa9. 00000020:00000010:3.0:1775764548.224307:0:6193:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at 000000006aaf5fa9. 00000400:00000010:2.0:1775764548.224556:0:6190:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000252feeb6. 00000400:00000200:2.0:1775764548.224564:0:6190:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c53a1658 00000400:00000010:2.0:1775764548.224566:0:6190:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 00000000c53a1658. 00000100:00000200:2.0:1775764548.224583:0:6190:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000c1e17336 x1862023861241600/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.224597:0:6190:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000c1e17336 x1862023861241600/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764548.224636:0:6190:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000f091afaf. 02000000:00000010:2.0:1775764548.224653:0:6190:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 00000000cc4821c3. 02000000:00000010:2.0:1775764548.224680:0:6190:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000003884c2aa. 00000100:00000010:2.0:1775764548.224687:0:6190:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000c1e17336. 00000020:00000010:2.0:1775764548.224715:0:6190:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000dcf3a1f9. 00000020:00000010:2.0:1775764548.224721:0:6190:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000dcf3a1f9. 00000020:00000010:2.0:1775764548.224728:0:6190:0:(obd_config.c:1995:class_config_llog_handler()) kmalloced '(inst_name)': 28 at 000000005c41aff4. 00000020:00000010:2.0:1775764548.224733:0:6190:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 112 at 0000000036be77d4. 00000020:00000010:2.0:1775764548.224767:0:6190:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 112 at ffffa0e9c9bf1f00. 00000020:00000010:2.0:1775764548.224771:0:6190:0:(obd_config.c:2090:class_config_llog_handler()) kfreed 'inst_name': 28 at ffffa0e9c45601e0. 00000100:00000010:2.0:1775764548.224781:0:6190:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9d0710000. 02000000:00000010:2.0:1775764548.225633:0:6190:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffffa0e9d075be00. 00000100:00000010:2.0:1775764548.225654:0:6190:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffffa0e9d0759540. 02000000:00000010:2.0:1775764548.225698:0:6190:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffffa0e9c81f0000. 00000400:00000200:2.0:1775764548.225707:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764548.225711:0:6190:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c55e5b58. 00000400:00000010:2.0:1775764548.225717:0:6190:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffffa0e9d0759900. 00000100:00000200:2.0:1775764548.225725:0:6190:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1862023861241728, portal 25 00000400:00000010:2.0:1775764548.225732:0:6190:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d82003b8. 00000100:00000200:2.0:1775764548.225736:0:6190:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1862023861241728, offset 0 00000400:00000200:2.0:1775764548.225745:0:6190:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764548.225753:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.225761:0:6190:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764548.225766:0:6190:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764548.225770:0:6190:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764548.225774:0:6190:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764548.225777:0:6190:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764548.225783:0:6190:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [19:19:256:4294967295] 192.168.202.104@tcp LPNI seq info [19:19:8:4294967295] 00000400:00000200:2.0:1775764548.225789:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764548.225809:0:6190:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764548.225817:0:6190:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764548.225823:0:6190:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c84db900. 00000800:00000200:2.0:1775764548.225831:0:6190:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764548.225840:0:6190:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764548.225845:0:6190:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c84db900 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1775764548.225890:0:6190:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:2.0:1775764548.225896:0:6190:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffffa0e9c8de3d80. 00000100:00000010:2.0:1775764548.225912:0:6190:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffffa0e9c8de3c60. 00000800:00000200:1.0:1775764548.227051:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764548.227067:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c84db900. 00000400:00000200:1.0:1775764548.227073:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.227081:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764548.227088:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d82003b8 00000400:00000010:1.0:1775764548.227091:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d82003b8. 00000100:00000200:1.0:1775764548.227100:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9d0710000 x1862023861241728/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775764548.227557:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffffa0e9c8de3c60. 00000100:00000010:0.0:1775764548.230960:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffffa0e9c8de3d80. 00000020:00000010:1.0:1775764548.232247:0:6191:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffffa0e9c576da80. 00000020:00000010:1.0:1775764548.232260:0:6191:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffffa0e9c576da80. 00000400:00000200:1.0:1775764548.237404:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.237416:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.237426:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.237431:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.237438:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.237442:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 8472 into portal 25 MB=0x69d8036d00780 00000400:00000200:1.0:1775764548.237448:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.237454:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 8472/8472 into md 0x2b1 [4] + 224 00000400:00000200:1.0:1775764548.237479:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.237526:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.237553:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9d0710000 x1862023861241728/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775764548.237899:0:6190:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c55e5b58. 00000400:00000200:2.0:1775764548.237906:0:6190:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d0759900 00000400:00000010:2.0:1775764548.237909:0:6190:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffffa0e9d0759900. 00000100:00000200:2.0:1775764548.237915:0:6190:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9d0710000 x1862023861241728/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.237925:0:6190:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9d0710000 x1862023861241728/t0(0) o502->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 272/8472 e 0 to 0 dl 1775764564 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764548.237957:0:6190:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at ffffa0e9d0759540. 02000000:00000010:2.0:1775764548.237972:0:6190:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffffa0e9c81f0000. 02000000:00000010:2.0:1775764548.237988:0:6190:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffffa0e9d075be00. 00000100:00000010:2.0:1775764548.237993:0:6190:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9d0710000. 00000020:00000010:2.0:1775764548.238014:0:6190:0:(obd_config.c:2055:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffffa0e9c84db200. 00000020:00000010:2.0:1775764548.238020:0:6190:0:(obd_config.c:2088:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffffa0e9c84db200. 00000040:00000010:2.0:1775764548.238027:0:6190:0:(llog.c:894:llog_process_thread()) kvfree 'buf': 8192 at ffffa0e9d01a2000. 00000020:00000010:2.0:1775764548.238038:0:6190:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffffa0e9c2aa8dc0. 00000020:00000010:2.0:1775764548.238057:0:6190:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffffa0e9d075b200. 00000040:00000010:2.0:1775764548.238062:0:6190:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffffa0e9d075a600. 00000020:00000010:2.0:1775764548.238067:0:6190:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffffa0e9d04c9800. 00000020:00000010:2.0:1775764548.238074:0:6190:0:(tgt_main.c:740:tgt_key_fini()) slab-freed '(info)': 1288 at ffffa0e9c55bdf98. 00000020:00000010:2.0:1775764548.238082:0:6190:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffffa0e9d075ae00. 80000000:00000010:2.0:1775764548.238086:0:6190:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffffa0e9c8de3de0. 40000000:00000010:2.0:1775764548.238091:0:6190:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffffa0e9c2aa8e00. 00100000:00000010:2.0:1775764548.238096:0:6190:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffffa0e9d04ca000. 00040000:00000010:2.0:1775764548.238101:0:6190:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffffa0e9c84dbf00. 00040000:00000010:2.0:1775764548.238106:0:6190:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffffa0e9d04cd000. 00040000:00000010:2.0:1775764548.238110:0:6190:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffffa0e9d075b400. 00000004:00000010:2.0:1775764548.238114:0:6190:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffffa0e9d04cc000. 00000004:00000010:2.0:1775764548.238119:0:6190:0:(mdt_handler.c:8411:mdt_key_fini()) kfreed 'info': 2568 at ffffa0e9c8d1b000. 00000004:00000010:2.0:1775764548.238124:0:6190:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffffa0e9d04ca800. 00002000:00000010:2.0:1775764548.238137:0:6190:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffffa0e9c84a8000. 00000004:00000010:2.0:1775764548.238142:0:6190:0:(osp_dev.c:1823:osp_key_fini()) kfreed 'info': 616 at ffffa0e9c84a8400. 00000020:00000010:2.0:1775764548.238146:0:6190:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffffa0e9d075b800. 00000040:00000010:1.0:1775764548.238360:0:6160:0:(llog.c:989:llog_process_or_fork()) kfreed 'lpi': 96 at ffffa0e9d08f4de0. 00000040:00000010:1.0:1775764548.238368:0:6160:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at ffffa0e9d01a4000. 00000040:00000010:1.0:1775764548.238371:0:6160:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffffa0e9c9bd5a00. 00000020:00000010:1.0:1775764548.238378:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffffa0e9c5522a00. 00000040:00000010:1.0:1775764548.238382:0:6160:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffffa0e9c5523e00. 00000020:00000010:1.0:1775764548.238385:0:6160:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffffa0e9d071b800. 00000020:00000010:1.0:1775764548.238389:0:6160:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffffa0e9c5522c00. 80000000:00000010:1.0:1775764548.238392:0:6160:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffffa0e9c8539cc0. 00040000:00000010:1.0:1775764548.238395:0:6160:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffffa0e9c5523400. 20000000:00000010:1.0:1775764548.238405:0:6160:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffffa0e9c5522e00. 00000020:00000010:1.0:1775764548.238408:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffffa0e9c5522000. 10000000:00000010:1.0:1775764548.238411:0:6160:0:(mgc_request.c:1748:mgc_process_cfg_log()) kfreed 'env': 56 at ffffa0e9d0236340. 00000020:00000010:1.0:1775764548.238437:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffffa0e9c4e01e00. 00000040:00000010:1.0:1775764548.238442:0:6160:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffffa0e9c4e00600. 00000020:00000010:1.0:1775764548.238445:0:6160:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffffa0e9d09a6800. 00000020:00000010:1.0:1775764548.238448:0:6160:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffffa0e9c4e01000. 00040000:00000010:1.0:1775764548.238450:0:6160:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffffa0e9c4d46300. 00040000:00000010:1.0:1775764548.238455:0:6160:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffffa0e9d29dd200. 00000020:00000010:1.0:1775764548.238458:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffffa0e9c4e01200. 00000020:00000010:1.0:1775764548.238461:0:6160:0:(obd_mount.c:81:lustre_process_log()) kfreed 'lcfg': 184 at ffffa0e9c801ef00. 00000020:00000010:1.0:1775764548.238463:0:6160:0:(obd_mount.c:83:lustre_process_log()) kfreed 'bufs': 104 at ffffa0e9c4fffb00. 00000080:00000010:1.0:1775764548.238471:0:6160:0:(llite_lib.c:1471:ll_fill_super()) kmalloced '(dt)': 33 at ffffa0e9c576da80. 00000080:00000010:1.0:1775764548.238475:0:6160:0:(llite_lib.c:1477:ll_fill_super()) kmalloced '(md)': 33 at ffffa0e9c576d380. 00000080:00000010:1.0:1775764548.238482:0:6160:0:(llite_lib.c:346:client_common_fill_super()) kmalloced '(data)': 192 at ffffa0e9c4e15300. 00000080:00000010:1.0:1775764548.239008:0:6160:0:(llite_lib.c:350:client_common_fill_super()) kmalloced '(osfs)': 144 at ffffa0e9c4e15840. 00000020:00000010:1.0:1775764548.239017:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffffa0e9d07f8000. 00000020:00000010:1.0:1775764548.239067:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffffa0e9d07fe800. 00000400:00000200:1.0:1775764548.239087:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.239096:0:6160:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.104@tcp is discovered: rc = 1 00000100:00000010:1.0:1775764548.239118:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9c8d0df80. 02000000:00000010:1.0:1775764548.239138:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffffa0e9c8d0b000. 02000000:00000010:0.0:1775764548.239214:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9d0b41c00. 00000400:00000200:0.0:1775764548.239224:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:0.0:1775764548.239228:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c014b528. 00000400:00000010:0.0:1775764548.239235:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275770. 00000100:00000200:0.0:1775764548.239242:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861241856, portal 10 00000020:00000010:0.0:1775764548.239251:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9c840e000. 00000400:00000010:0.0:1775764548.239258:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275908. 00000100:00000200:0.0:1775764548.239261:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862023861241856, offset 0 00000400:00000200:0.0:1775764548.239270:0:2400:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:0.0:1775764548.239278:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.239286:0:2400:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764548.239292:0:2400:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775764548.239298:0:2400:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:0.0:1775764548.239302:0:2400:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775764548.239305:0:2400:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:0.0:1775764548.239309:0:2400:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [20:20:256:4294967295] 192.168.202.104@tcp LPNI seq info [20:20:8:4294967295] 00000400:00000200:0.0:1775764548.239313:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.239324:0:2400:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:0.0:1775764548.239331:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764548.239337:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c9bbc900. 00000800:00000200:0.0:1775764548.239345:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:0.0:1775764548.239355:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764548.239368:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c9bbc900 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1775764548.239388:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:0.0:1775764548.239393:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffffa0e9d0613060. 00000100:00000010:0.0:1775764548.239409:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffffa0e9d0613a80. 40000000:00000010:1.0:1775764548.239465:0:6160:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffffa0e9c3144b00. 40000000:00000010:1.0:1775764548.239469:0:6160:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffffa0e9c4e15000. 40000000:00000010:1.0:1775764548.239617:0:6160:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffffa0e9c4e15000. 80000000:00000010:1.0:1775764548.239625:0:6160:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffffa0e9c576d140. 00020000:00000010:1.0:1775764548.239634:0:6160:0:(lu_tgt_descs.c:102:lu_qos_add_tgt()) kmalloced '(svr)': 104 at ffffa0e9c9b83980. 00000100:00000010:1.0:1775764548.239643:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9c8d0f480. 02000000:00000010:1.0:1775764548.239650:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffffa0e9c3144400. 00000020:00000010:1.0:1775764548.239692:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffffa0e9d07fb000. 00000400:00000200:1.0:1775764548.239703:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.239706:0:6160:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.104@tcp is discovered: rc = 1 00000100:00000010:1.0:1775764548.239714:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9c8d0e680. 02000000:00000010:1.0:1775764548.239727:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffffa0e9c8d0a000. 02000000:00000010:0.0:1775764548.242074:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9d0b41000. 00000400:00000200:0.0:1775764548.242097:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:0.0:1775764548.242105:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c014b948. 00000400:00000010:0.0:1775764548.242113:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275dd0. 00000100:00000200:0.0:1775764548.242122:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861242112, portal 10 00000020:00000010:0.0:1775764548.242133:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9c840a000. 00000400:00000010:0.0:1775764548.242141:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275b28. 00000100:00000200:0.0:1775764548.242145:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862023861242112, offset 0 00000400:00000200:0.0:1775764548.242155:0:2400:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:0.0:1775764548.242164:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.242174:0:2400:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764548.242180:0:2400:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775764548.242184:0:2400:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:0.0:1775764548.242188:0:2400:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775764548.242191:0:2400:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:0.0:1775764548.242196:0:2400:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [21:21:255:4294967295] 192.168.202.104@tcp LPNI seq info [21:21:7:4294967295] 00000400:00000200:0.0:1775764548.242200:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764548.242211:0:2400:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:0.0:1775764548.242218:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764548.242225:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c9bbc800. 00000800:00000200:0.0:1775764548.242234:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:0.0:1775764548.242245:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764548.242250:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c9bbc800 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1775764548.242258:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000020:00000010:0.0:1775764548.249879:0:2401:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at ffffa0e9d07dda90. 00000020:00000010:0.0:1775764548.249895:0:2401:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffffa0e9d02b9a00. 80000000:00000010:0.0:1775764548.249912:0:2401:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffffa0e9d06133c0. 40000000:00000010:0.0:1775764548.249918:0:2401:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffffa0e9d0fc3200. 00000008:00000010:0.0:1775764548.249926:0:2401:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d098e468. 00020000:00000010:0.0:1775764548.249932:0:2401:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9d83c3480. 00008000:00000010:0.0:1775764548.249939:0:2401:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9c456d1a8. 00100000:00000010:0.0:1775764548.249947:0:2401:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffffa0e9d07f2800. 00040000:00000010:0.0:1775764548.249952:0:2401:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffffa0e9c9bbc000. 00040000:00000010:0.0:1775764548.249957:0:2401:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffffa0e9d07f4000. 00040000:00000010:0.0:1775764548.249962:0:2401:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffffa0e9d02b8000. 00000004:00000010:0.0:1775764548.249967:0:2401:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffffa0e9d07f5000. 00000004:00000010:0.0:1775764548.249976:0:2401:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at ffffa0e9c7d3e000. 00000004:00000010:0.0:1775764548.249981:0:2401:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffffa0e9d07f6000. 00002000:00000010:0.0:1775764548.249986:0:2401:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffffa0e9d0b42000. 00000004:00000010:0.0:1775764548.249991:0:2401:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at ffffa0e9d0b43000. 00000080:00000010:0.0:1775764548.249997:0:2401:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9d02c32a0. 00000080:00000010:0.0:1775764548.250001:0:2401:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9c5768ff8. 00000008:00000010:0.0:1775764548.250007:0:2401:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9c8dcb340. 00020000:00000010:0.0:1775764548.250010:0:2401:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9c8d9ed00. 00008000:00000010:0.0:1775764548.250013:0:2401:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9c8dc9e98. 00000080:00000010:0.0:1775764548.250017:0:2401:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d8274f00. 00000020:00000010:0.0:1775764548.250051:0:2402:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at ffffa0e9d07da338. 00000020:00000010:0.0:1775764548.250056:0:2402:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffffa0e9d02b8e00. 80000000:00000010:0.0:1775764548.250058:0:2402:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffffa0e9d0613a20. 40000000:00000010:0.0:1775764548.250061:0:2402:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffffa0e9d0fc3180. 00000008:00000010:0.0:1775764548.250065:0:2402:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d0988b28. 00020000:00000010:0.0:1775764548.250068:0:2402:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9d83c3a80. 00008000:00000010:0.0:1775764548.250071:0:2402:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9c456dc40. 00100000:00000010:0.0:1775764548.250094:0:2402:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffffa0e9c81f8000. 00040000:00000010:0.0:1775764548.250098:0:2402:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffffa0e9c9bbce00. 00040000:00000010:0.0:1775764548.250102:0:2402:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffffa0e9c81fe800. 00040000:00000010:0.0:1775764548.250104:0:2402:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffffa0e9d02b9c00. 00000004:00000010:0.0:1775764548.250107:0:2402:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffffa0e9c81fb000. 00000004:00000010:0.0:1775764548.250111:0:2402:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at ffffa0e9d05a1000. 00000004:00000010:0.0:1775764548.250119:0:2402:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffffa0e9c81f9000. 00002000:00000010:0.0:1775764548.250121:0:2402:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffffa0e9d0b41400. 00000004:00000010:0.0:1775764548.250125:0:2402:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at ffffa0e9d0b42800. 00000080:00000010:0.0:1775764548.250127:0:2402:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9d02c39d8. 00000080:00000010:0.0:1775764548.250130:0:2402:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9c5768000. 00000008:00000010:0.0:1775764548.250135:0:2402:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9c8dcadc0. 00020000:00000010:0.0:1775764548.250138:0:2402:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9c8d9e640. 00008000:00000010:0.0:1775764548.250140:0:2402:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9c8dc99a0. 00000080:00000010:0.0:1775764548.250143:0:2402:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d82745a0. 00000800:00000200:0.0:1775764548.250205:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.250209:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c9bbc900. 00000400:00000200:0.0:1775764548.250214:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.250227:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.250236:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275908 00000400:00000010:0.0:1775764548.250239:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275908. 00000100:00000200:0.0:1775764548.250254:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9c8d0df80 x1862023861241856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775764548.253888:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764548.253902:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c9bbc800. 00000400:00000200:0.0:1775764548.253913:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764548.253924:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764548.253934:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275b28 00000400:00000010:0.0:1775764548.253936:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275b28. 00000100:00000200:0.0:1775764548.253953:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9c8d0e680 x1862023861242112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 40000000:00000010:1.0:1775764548.254144:0:6160:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffffa0e9c3144e00. 40000000:00000010:1.0:1775764548.254151:0:6160:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffffa0e9c4e15000. 40000000:00000010:1.0:1775764548.254223:0:6160:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffffa0e9c4e15000. 80000000:00000010:1.0:1775764548.254230:0:6160:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffffa0e9c576d980. 00000100:00000010:1.0:1775764548.254246:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9c8d0d500. 02000000:00000010:1.0:1775764548.254263:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffffa0e9c3144000. 00800000:00000010:1.0:1775764548.254312:0:6160:0:(lmv_obd.c:1420:lmv_statfs()) kmalloced '(temp)': 144 at ffffa0e9c4e15000. 00000100:00000010:1.0:1775764548.254319:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9c8d0ea00. 02000000:00000010:1.0:1775764548.254323:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffffa0e9c3144d00. 00000100:00000010:1.0:1775764548.254332:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffffa0e9c4e15b40. 00000100:00000010:1.0:1775764548.254383:0:447:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffffa0e9d0613a80. 00000100:00000010:0.0:1775764548.255732:0:706:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffffa0e9d0613060. 00000400:00000200:1.0:1775764548.260198:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260209:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.260219:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260225:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260232:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260237:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 416 into portal 10 MB=0x69d8036d00800 00000400:00000200:1.0:1775764548.260243:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260250:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 416/416 into md 0x2b9 [1] + 0 00000400:00000200:1.0:1775764548.260265:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.260270:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.260287:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9c8d0df80 x1862023861241856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775764548.260329:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260332:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764548.260336:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260339:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260342:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260344:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 416 into portal 10 MB=0x69d8036d00900 00000400:00000200:1.0:1775764548.260348:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764548.260363:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 416/416 into md 0x2c9 [1] + 0 00000400:00000200:1.0:1775764548.260386:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764548.260390:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764548.260399:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9c8d0e680 x1862023861242112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775764548.260474:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c014b948. 00000400:00000200:2.0:1775764548.260480:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275dd0 00000400:00000010:2.0:1775764548.260520:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275dd0. 00000100:00000200:2.0:1775764548.260529:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9c8d0e680 x1862023861242112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764548.260546:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9c8d0e680 x1862023861242112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775764548.260572:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9d01a2000. 00000020:00000010:3.0:1775764549.036466:0:126:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffffa0e9c876f400. 00000400:00000010:0.0:1775764549.271227:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c014b528. 00000400:00000200:0.0:1775764549.271238:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275770 00000400:00000010:0.0:1775764549.271241:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275770. 00000100:00000200:0.0:1775764549.271258:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9c8d0df80 x1862023861241856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775764549.271270:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9c8d0df80 x1862023861241856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 1775764558 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775764549.271451:0:2401:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9c8d09400. 00000400:00000200:1.0:1775764549.271463:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:1.0:1775764549.271468:0:2401:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9d0b549a0. 00000400:00000010:1.0:1775764549.271473:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d09dbf68. 00000100:00000200:1.0:1775764549.271481:0:2401:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861242240, portal 10 00000020:00000010:1.0:1775764549.271521:0:2401:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9c8d54000. 00000400:00000010:1.0:1775764549.271528:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d09db660. 00000100:00000200:1.0:1775764549.271531:0:2401:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1862023861242240, offset 0 00000400:00000200:1.0:1775764549.271538:0:2401:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:1.0:1775764549.271548:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.271556:0:2401:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:1.0:1775764549.271561:0:2401:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775764549.271566:0:2401:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:1.0:1775764549.271580:0:2401:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775764549.271583:0:2401:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:1.0:1775764549.271587:0:2401:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [22:22:256:4294967295] 192.168.202.104@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:1.0:1775764549.271591:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.271601:0:2401:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:1.0:1775764549.271607:0:2401:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:1.0:1775764549.271611:0:2401:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c3144a00. 00000800:00000200:1.0:1775764549.271618:0:2401:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:1.0:1775764549.271626:0:2401:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:1.0:1775764549.271630:0:2401:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c3144a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1775764549.271651:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000100:00000010:1.0:1775764549.271656:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffffa0e9c8539cc0. 00000100:00000010:1.0:1775764549.271687:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffffa0e9d04eacc0. 02000000:00000010:1.0:1775764549.271699:0:2401:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9c8d08000. 00000400:00000200:1.0:1775764549.271702:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:1.0:1775764549.271705:0:2401:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9d0b54790. 00000400:00000010:1.0:1775764549.271708:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d09db550. 00000100:00000200:1.0:1775764549.271710:0:2401:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861241984, portal 10 00000020:00000010:1.0:1775764549.271715:0:2401:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9c8d50000. 00000400:00000010:1.0:1775764549.271719:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d09dbbb0. 00000100:00000200:1.0:1775764549.271720:0:2401:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1862023861241984, offset 0 00000400:00000200:1.0:1775764549.271725:0:2401:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:1.0:1775764549.271729:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.271732:0:2401:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:1.0:1775764549.271735:0:2401:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775764549.271737:0:2401:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:1.0:1775764549.271739:0:2401:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775764549.271742:0:2401:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:1.0:1775764549.271744:0:2401:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [23:23:255:4294967295] 192.168.202.104@tcp LPNI seq info [23:23:7:4294967295] 00000400:00000200:1.0:1775764549.271748:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.271753:0:2401:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:1.0:1775764549.271757:0:2401:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:1.0:1775764549.271761:0:2401:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c3144c00. 00000800:00000200:1.0:1775764549.271764:0:2401:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:1.0:1775764549.271768:0:2401:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:1.0:1775764549.271771:0:2401:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c3144c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1775764549.271776:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:0.0:1775764549.272466:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764549.272471:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c3144a00. 00000400:00000200:0.0:1775764549.272476:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764549.272533:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764549.272539:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d09db660 00000400:00000010:0.0:1775764549.272542:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d09db660. 00000100:00000200:0.0:1775764549.272551:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9c8d0d500 x1862023861242240/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775764549.272639:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764549.272641:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c3144c00. 00000400:00000200:0.0:1775764549.272644:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764549.272647:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764549.272657:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d09dbbb0 00000400:00000010:0.0:1775764549.272659:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d09dbbb0. 00000100:00000200:0.0:1775764549.272670:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9c8d0f480 x1862023861241984/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775764549.276226:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276236:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.276245:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276251:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276257:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276262:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 368 into portal 10 MB=0x69d8036d00980 00000400:00000200:1.0:1775764549.276285:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276292:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 368/368 into md 0x2d9 [1] + 224 00000400:00000200:1.0:1775764549.276307:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.276311:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.276325:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9c8d0d500 x1862023861242240/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775764549.276357:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276360:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.276364:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276367:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276369:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276372:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 368 into portal 10 MB=0x69d8036d00880 00000400:00000200:1.0:1775764549.276375:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.276379:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 368/368 into md 0x2e9 [1] + 224 00000400:00000200:1.0:1775764549.276389:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.276392:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.276398:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9c8d0f480 x1862023861241984/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775764549.276426:0:2401:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9d0b549a0. 00000400:00000200:1.0:1775764549.276431:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d09dbf68 00000400:00000010:1.0:1775764549.276434:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d09dbf68. 00000100:00000200:1.0:1775764549.276439:0:2401:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9c8d0d500 x1862023861242240/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775764549.276448:0:2401:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9c8d0d500 x1862023861242240/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775764549.276918:0:2401:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9d0b54790. 00000400:00000200:1.0:1775764549.276925:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d09db550 00000400:00000010:1.0:1775764549.276926:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d09db550. 00000100:00000200:1.0:1775764549.276933:0:2401:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9c8d0f480 x1862023861241984/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775764549.276951:0:2401:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9c8d0f480 x1862023861241984/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775764549.276973:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9c8d09400. 02000000:00000010:1.0:1775764549.276980:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffffa0e9c3144000. 00000100:00000010:1.0:1775764549.276987:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9c8d0d500. 02000000:00000010:1.0:1775764549.276991:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9c8d08000. 02000000:00000010:1.0:1775764549.276993:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffffa0e9c3144400. 00000100:00000010:1.0:1775764549.276995:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9c8d0f480. 02000000:00000010:2.0:1775764549.277109:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9d0b41000. 02000000:00000010:2.0:1775764549.277115:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffffa0e9c8d0a000. 00000100:00000010:2.0:1775764549.277118:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9c8d0e680. 02000000:00000010:2.0:1775764549.277122:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9d0b41c00. 02000000:00000010:2.0:1775764549.277124:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffffa0e9c8d0b000. 00000100:00000010:2.0:1775764549.277127:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9c8d0df80. 02000000:00000010:1.0:1775764549.277277:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9c8d08000. 00000400:00000200:1.0:1775764549.277285:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:1.0:1775764549.277288:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9d0b54790. 00000400:00000010:1.0:1775764549.277293:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d09db550. 00000100:00000200:1.0:1775764549.277298:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861242368, portal 10 00000400:00000010:1.0:1775764549.277303:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d09dbf68. 00000100:00000200:1.0:1775764549.277306:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1862023861242368, offset 0 00000400:00000200:1.0:1775764549.277312:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:1.0:1775764549.277318:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.277325:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:1.0:1775764549.277330:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775764549.277334:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:1.0:1775764549.277338:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775764549.277341:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:1.0:1775764549.277345:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [24:24:256:4294967295] 192.168.202.104@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:1.0:1775764549.277349:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.277358:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:1.0:1775764549.277363:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:1.0:1775764549.277368:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c3144400. 00000800:00000200:1.0:1775764549.277374:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:1.0:1775764549.277381:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:1.0:1775764549.277392:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c3144400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1775764549.277414:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:0.0:1775764549.278093:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1775764549.278099:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c3144400. 00000400:00000200:0.0:1775764549.278103:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1775764549.278109:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:0.0:1775764549.278113:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d09dbf68 00000400:00000010:0.0:1775764549.278115:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d09dbf68. 00000100:00000200:0.0:1775764549.278123:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9c8d0ea00 x1862023861242368/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775764549.306078:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.306089:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.306101:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.306107:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.306113:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.306117:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 368 into portal 10 MB=0x69d8036d00a00 00000400:00000200:1.0:1775764549.306124:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.306130:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 368/368 into md 0x2f9 [1] + 224 00000400:00000200:1.0:1775764549.306151:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.306155:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.306169:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9c8d0ea00 x1862023861242368/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775764549.312380:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9d0b54790. 00000400:00000200:2.0:1775764549.312399:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d09db550 00000400:00000010:2.0:1775764549.312401:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d09db550. 00000100:00000200:2.0:1775764549.312421:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9c8d0ea00 x1862023861242368/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764549.312436:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9c8d0ea00 x1862023861242368/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775764549.312469:0:6160:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9d01a4000. 00000100:00000010:2.0:1775764549.313112:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at ffffa0e9c4e15b40. 02000000:00000010:2.0:1775764549.313135:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9c8d08000. 02000000:00000010:2.0:1775764549.313142:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffffa0e9c3144d00. 00000100:00000010:2.0:1775764549.313148:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9c8d0ea00. 00800000:00000010:2.0:1775764549.313157:0:6160:0:(lmv_obd.c:1480:lmv_statfs()) kfreed 'temp': 144 at ffffa0e9c4e15000. 00000100:00000010:2.0:1775764549.313180:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9d0710000. 02000000:00000010:2.0:1775764549.313217:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffffa0e9d075a400. 00000100:00000010:2.0:1775764549.313237:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffffa0e9d0759540. 02000000:00000010:2.0:1775764549.313266:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9c8131000. 00000400:00000200:2.0:1775764549.313277:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764549.313283:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c55e5b58. 00000400:00000010:2.0:1775764549.313289:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8200110. 00000100:00000200:2.0:1775764549.313298:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861242496, portal 10 00000400:00000010:2.0:1775764549.313305:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8200220. 00000100:00000200:2.0:1775764549.313308:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 440 bytes to portal 12, xid 1862023861242496, offset 0 00000400:00000200:2.0:1775764549.313318:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764549.313326:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.313337:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764549.313344:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764549.313349:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764549.313353:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764549.313356:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764549.313361:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [25:25:256:4294967295] 192.168.202.104@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:2.0:1775764549.313366:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.313378:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764549.313385:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 440 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764549.313391:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c84dbf00. 00000800:00000200:2.0:1775764549.313400:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764549.313410:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764549.313416:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c84dbf00 type 1, nob 536 niov 1 nkiov 1 00000400:00000200:2.0:1775764549.313523:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:1.0:1775764549.325053:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764549.325065:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c84dbf00. 00000400:00000200:1.0:1775764549.325076:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.325087:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764549.325097:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8200220 00000400:00000010:1.0:1775764549.325100:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8200220. 00000100:00000200:1.0:1775764549.325114:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9d0710000 x1862023861242496/t0(0) o40->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 440/448 e 0 to 0 dl 1775764565 ref 3 fl Rpc:r/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775764549.331173:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.331182:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.331191:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.331196:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.331202:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.331205:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 448 into portal 10 MB=0x69d8036d00a80 00000400:00000200:1.0:1775764549.331211:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.331216:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 448/448 into md 0x309 [1] + 224 00000400:00000200:1.0:1775764549.331248:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.331252:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.331266:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9d0710000 x1862023861242496/t0(0) o40->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 440/448 e 0 to 0 dl 1775764565 ref 2 fl Rpc:Qr/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775764549.332442:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c55e5b58. 00000400:00000200:2.0:1775764549.332452:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8200110 00000400:00000010:2.0:1775764549.332455:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8200110. 00000100:00000200:2.0:1775764549.332467:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9d0710000 x1862023861242496/t0(0) o40->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 440/448 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQ/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764549.332478:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9d0710000 x1862023861242496/t0(0) o40->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 440/448 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQU/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764549.333374:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at ffffa0e9d0759540. 00000002:00000200:2.0:1775764549.333386:0:6160:0:(mdc_request.c:134:mdc_get_root()) root fid=[0x200000007:0x1:0x0], last_committed=0 02000000:00000010:2.0:1775764549.333394:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9c8131000. 02000000:00000010:2.0:1775764549.333400:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffffa0e9d075a400. 00000100:00000010:2.0:1775764549.333405:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9d0710000. 00000020:00000010:2.0:1775764549.333437:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffffa0e9d04ca800. 00000020:00000010:2.0:1775764549.333465:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffffa0e9d04cc000. 00000400:00000200:2.0:1775764549.333517:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.333537:0:6160:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.104@tcp is discovered: rc = 1 00000100:00000010:2.0:1775764549.333560:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9d0710000. 02000000:00000010:2.0:1775764549.333646:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffffa0e9c8131000. 02000000:00000010:0.0:1775764549.334012:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9d0b41000. 00000400:00000200:0.0:1775764549.334021:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:0.0:1775764549.334025:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c014b528. 00000400:00000010:0.0:1775764549.334031:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275770. 00000100:00000200:0.0:1775764549.334038:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861242624, portal 4 00000020:00000010:0.0:1775764549.334050:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9c8408000. 00000400:00000010:0.0:1775764549.334073:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275b28. 00000100:00000200:0.0:1775764549.334077:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862023861242624, offset 0 00000400:00000200:0.0:1775764549.334085:0:2400:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:0.0:1775764549.334093:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764549.334101:0:2400:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764549.334107:0:2400:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775764549.334112:0:2400:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:0.0:1775764549.334116:0:2400:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775764549.334119:0:2400:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:0.0:1775764549.334123:0:2400:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [26:26:256:4294967295] 192.168.202.104@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:0.0:1775764549.334128:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764549.334138:0:2400:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:0.0:1775764549.334144:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764549.334149:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c9bbc800. 00000800:00000200:0.0:1775764549.334156:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:0.0:1775764549.334165:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764549.334170:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c9bbc800 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1775764549.334196:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000020:00000010:2.0:1775764549.334297:0:6160:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffffa0e9d04cd000. 00000400:00000200:2.0:1775764549.334310:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.334314:0:6160:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.104@tcp is discovered: rc = 1 00000100:00000010:2.0:1775764549.334323:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9d0712d80. 02000000:00000010:2.0:1775764549.334330:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffffa0e9c8130800. 00000100:00000010:3.0:1775764549.334387:0:38:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9d0541880. 02000000:00000010:3.0:1775764549.334395:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffffa0e9c4d46f00. 00000020:00000010:3.0:1775764549.334435:0:2404:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at ffffa0e9c2c12338. 00000020:00000010:3.0:1775764549.334441:0:2404:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffffa0e9d31ae200. 80000000:00000010:3.0:1775764549.334445:0:2404:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffffa0e9c54695a0. 40000000:00000010:3.0:1775764549.334449:0:2404:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffffa0e9d0236ac0. 00000008:00000010:3.0:1775764549.334454:0:2404:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d04f42f0. 00020000:00000010:3.0:1775764549.334459:0:2404:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9e06c8240. 00008000:00000010:3.0:1775764549.334463:0:2404:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9d04edfc8. 00100000:00000010:3.0:1775764549.334468:0:2404:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffffa0e9d071f000. 00040000:00000010:3.0:1775764549.334472:0:2404:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffffa0e9c4d46300. 00040000:00000010:3.0:1775764549.334501:0:2404:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffffa0e9d071c800. 00040000:00000010:3.0:1775764549.334519:0:2404:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffffa0e9c8da4a00. 00000004:00000010:3.0:1775764549.334525:0:2404:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffffa0e9d071d800. 00000004:00000010:3.0:1775764549.334530:0:2404:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at ffffa0e9c8d14000. 00000004:00000010:3.0:1775764549.334534:0:2404:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffffa0e9d0718000. 00002000:00000010:3.0:1775764549.334538:0:2404:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffffa0e9d01c4c00. 00000004:00000010:3.0:1775764549.334543:0:2404:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at ffffa0e9d01c4000. 00000080:00000010:3.0:1775764549.334548:0:2404:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9d01cda80. 00000080:00000010:3.0:1775764549.334553:0:2404:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9c8482da0. 00000008:00000010:3.0:1775764549.334560:0:2404:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9d04e7600. 00020000:00000010:3.0:1775764549.334564:0:2404:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9c841b180. 00008000:00000010:3.0:1775764549.334580:0:2404:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9d01cb830. 00000080:00000010:3.0:1775764549.334584:0:2404:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d01ca2d0. 02000000:00000010:3.0:1775764549.334593:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffffa0e9c8da4000. 00000400:00000200:3.0:1775764549.334597:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:3.0:1775764549.334601:0:2404:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9d2a15bb0. 00000400:00000010:3.0:1775764549.334606:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9c4e3b440. 00000100:00000200:3.0:1775764549.334610:0:2404:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1862023861242880, portal 25 00000400:00000010:3.0:1775764549.334614:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9c4e3b4c8. 00000100:00000200:3.0:1775764549.334617:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1862023861242880, offset 0 00000400:00000200:3.0:1775764549.334622:0:2404:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:3.0:1775764549.334626:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764549.334630:0:2404:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:3.0:1775764549.334633:0:2404:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775764549.334635:0:2404:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:3.0:1775764549.334638:0:2404:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775764549.334640:0:2404:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:3.0:1775764549.334643:0:2404:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [27:27:255:4294967295] 192.168.202.104@tcp LPNI seq info [27:27:7:4294967295] 00000400:00000200:3.0:1775764549.334647:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:3.0:1775764549.334671:0:2404:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:3.0:1775764549.334676:0:2404:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:3.0:1775764549.334680:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c4d46100. 00000800:00000200:3.0:1775764549.334690:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:3.0:1775764549.334695:0:2404:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:3.0:1775764549.334698:0:2404:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c4d46100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1775764549.334703:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000020:00000010:3.0:1775764549.334741:0:2403:0:(tgt_main.c:720:tgt_key_init()) slab-alloced 'thread': 1288 at ffffa0e9c2c169a8. 00000020:00000010:3.0:1775764549.334745:0:2403:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffffa0e9c8da4c00. 80000000:00000010:3.0:1775764549.334748:0:2403:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffffa0e9c5469780. 40000000:00000010:3.0:1775764549.334751:0:2403:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffffa0e9d02365c0. 00000008:00000010:3.0:1775764549.334755:0:2403:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d04f37c8. 00020000:00000010:3.0:1775764549.334758:0:2403:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9e06c8f00. 00008000:00000010:3.0:1775764549.334761:0:2403:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9d04ed8b8. 00100000:00000010:3.0:1775764549.334764:0:2403:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffffa0e9d071b800. 00040000:00000010:3.0:1775764549.334767:0:2403:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffffa0e9c4d46400. 00040000:00000010:3.0:1775764549.334771:0:2403:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffffa0e9d019b000. 00040000:00000010:3.0:1775764549.334773:0:2403:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffffa0e9c8da5a00. 00000004:00000010:3.0:1775764549.334776:0:2403:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffffa0e9d09a6800. 00000800:00000200:1.0:1775764549.335472:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764549.335479:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c9bbc800. 00000400:00000200:1.0:1775764549.335506:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.335513:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764549.335519:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275b28 00000400:00000010:1.0:1775764549.335522:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275b28. 00000100:00000200:1.0:1775764549.335543:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9d0710000 x1862023861242624/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775764549.335649:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764549.335651:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c4d46100. 00000400:00000200:1.0:1775764549.335656:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.335660:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764549.335662:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9c4e3b4c8 00000400:00000010:1.0:1775764549.335664:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9c4e3b4c8. 00000100:00000200:1.0:1775764549.335669:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9d0541880 x1862023861242880/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000004:00000010:3.0:1775764549.339480:0:2403:0:(mdt_handler.c:8386:mdt_key_init()) kmalloced '(value)': 2568 at ffffa0e9c8d11000. 00000004:00000010:3.0:1775764549.339516:0:2403:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffffa0e9d09a3000. 00002000:00000010:3.0:1775764549.339521:0:2403:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffffa0e9d01c5400. 00000004:00000010:3.0:1775764549.339535:0:2403:0:(osp_dev.c:1823:osp_key_init()) kmalloced '(value)': 616 at ffffa0e9d01c7000. 00000080:00000010:3.0:1775764549.339539:0:2403:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9d01cd690. 00000080:00000010:3.0:1775764549.339543:0:2403:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9c8480248. 00000008:00000010:3.0:1775764549.339551:0:2403:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9d04e7080. 00020000:00000010:3.0:1775764549.339556:0:2403:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9c8419440. 00008000:00000010:3.0:1775764549.339560:0:2403:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9d01cb488. 00000080:00000010:3.0:1775764549.339564:0:2403:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d01ca4b0. 00000400:00000200:1.0:1775764549.344644:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.344666:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.344675:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.344681:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.344687:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.344691:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 416 into portal 4 MB=0x69d8036d00b00 00000400:00000200:1.0:1775764549.344697:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.344703:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.104@tcp of length 416/416 into md 0x311 [1] + 0 00000400:00000200:1.0:1775764549.344718:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.344723:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.344740:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9d0710000 x1862023861242624/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775764549.345311:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffffa0e9d0b40000. 00000400:00000200:0.0:1775764549.345319:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:0.0:1775764549.345323:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c014bf20. 00000400:00000010:0.0:1775764549.345329:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275908. 00000100:00000200:0.0:1775764549.345337:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023861242752, portal 4 00000020:00000010:0.0:1775764549.345364:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffffa0e9c8dfa000. 00000400:00000010:0.0:1775764549.345372:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8275000. 00000100:00000200:0.0:1775764549.345375:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862023861242752, offset 0 00000400:00000200:0.0:1775764549.345382:0:2400:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:0.0:1775764549.345389:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764549.345396:0:2400:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:0.0:1775764549.345401:0:2400:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775764549.345406:0:2400:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:0.0:1775764549.345415:0:2400:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775764549.345419:0:2400:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:0.0:1775764549.345423:0:2400:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [28:28:256:4294967295] 192.168.202.104@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:0.0:1775764549.345428:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:0.0:1775764549.345437:0:2400:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:0.0:1775764549.345443:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:0.0:1775764549.345449:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c9bbc900. 00000800:00000200:0.0:1775764549.345455:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:0.0:1775764549.345462:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:0.0:1775764549.345467:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c9bbc900 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1775764549.345515:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000080:00000010:2.0:1775764549.351187:0:6160:0:(llite_lib.c:736:client_common_fill_super()) kmalloced '(op_data)': 384 at ffffa0e9d075a400. 00000020:00000010:2.0:1775764549.351207:0:6160:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffffa0e9c8133800. 00000100:00000010:2.0:1775764549.351223:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9d0710380. 02000000:00000010:2.0:1775764549.351239:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffffa0e9d075a800. 00000100:00000010:2.0:1775764549.351257:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffffa0e9d0759540. 02000000:00000010:2.0:1775764549.351292:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffffa0e9d075b600. 00000400:00000200:2.0:1775764549.351304:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764549.351309:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c55e5b58. 00000400:00000010:2.0:1775764549.351316:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8200110. 00000100:00000200:2.0:1775764549.351324:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1862023861243008, portal 10 00000400:00000010:2.0:1775764549.351331:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8200f68. 00000100:00000200:2.0:1775764549.351336:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 264 bytes to portal 29, xid 1862023861243008, offset 0 00000400:00000200:2.0:1775764549.351348:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764549.351359:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.351370:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764549.351395:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764549.351401:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764549.351406:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764549.351409:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764549.351414:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [29:29:255:4294967295] 192.168.202.104@tcp LPNI seq info [29:29:7:4294967295] 00000400:00000200:2.0:1775764549.351419:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.351431:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764549.351438:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 264 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764549.351444:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c84db200. 00000800:00000200:2.0:1775764549.351453:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764549.351461:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764549.351466:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c84db200 type 1, nob 360 niov 1 nkiov 1 00000400:00000200:2.0:1775764549.351475:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:1.0:1775764549.352198:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764549.352204:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c9bbc900. 00000400:00000200:1.0:1775764549.352209:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.352216:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764549.352221:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275000 00000400:00000010:1.0:1775764549.352224:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275000. 00000100:00000200:1.0:1775764549.352234:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9d0712d80 x1862023861242752/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775764549.353105:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.353113:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.353121:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.353125:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.353130:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.353135:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 224 into portal 25 MB=0x69d8036d00c00 00000400:00000200:1.0:1775764549.353140:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.353146:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.104@tcp of length 224/224 into md 0x321 [1] + 224 00000400:00000200:1.0:1775764549.353163:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.353167:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.353177:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9d0541880 x1862023861242880/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764565 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775764549.353347:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764549.353351:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c84db200. 00000400:00000200:1.0:1775764549.353355:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.353358:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764549.353361:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8200f68 00000400:00000010:1.0:1775764549.353364:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8200f68. 00000100:00000200:1.0:1775764549.353370:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9d0710380 x1862023861243008/t0(0) o900->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:29/10 lens 264/248 e 0 to 0 dl 1775764565 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775764549.359627:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c014b528. 00000400:00000200:0.0:1775764549.359642:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275770 00000400:00000010:0.0:1775764549.359645:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275770. 00000100:00000200:0.0:1775764549.359671:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9d0710000 x1862023861242624/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775764549.359683:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9d0710000 x1862023861242624/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775764549.359774:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9d0b41000. 02000000:00000010:0.0:1775764549.359966:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffffa0e9c8131000. 00000100:00000010:0.0:1775764549.359974:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9d0710000. 00000400:00000010:3.0:1775764549.364213:0:2404:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9d2a15bb0. 00000400:00000200:3.0:1775764549.364228:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9c4e3b440 00000400:00000010:3.0:1775764549.364231:0:2404:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9c4e3b440. 00000100:00000200:3.0:1775764549.364249:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9d0541880 x1862023861242880/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764565 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775764549.364261:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9d0541880 x1862023861242880/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764565 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775764549.364318:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffffa0e9c8da4000. 02000000:00000010:3.0:1775764549.364324:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffffa0e9c4d46f00. 00000100:00000010:3.0:1775764549.364332:0:2404:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9d0541880. 00000400:00000200:1.0:1775764549.368898:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.368929:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.368938:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.368943:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.368948:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.368952:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 416 into portal 4 MB=0x69d8036d00b80 00000400:00000200:1.0:1775764549.368958:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.368964:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.202.104@tcp of length 416/416 into md 0x329 [1] + 0 00000400:00000200:1.0:1775764549.368981:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.368987:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.369002:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9d0712d80 x1862023861242752/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775764549.369040:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.369043:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.369047:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.369049:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.369052:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.369056:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 248 into portal 10 MB=0x69d8036d00c80 00000400:00000200:1.0:1775764549.369060:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.369064:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 248/248 into md 0x339 [1] + 224 00000400:00000200:1.0:1775764549.369088:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.369092:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.369099:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9d0710380 x1862023861243008/t0(0) o900->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:29/10 lens 264/248 e 0 to 0 dl 1775764565 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775764549.369248:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c014bf20. 00000400:00000200:0.0:1775764549.369253:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8275908 00000400:00000010:0.0:1775764549.369256:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8275908. 00000100:00000200:0.0:1775764549.369262:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9d0712d80 x1862023861242752/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775764549.369271:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9d0712d80 x1862023861242752/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 1775764559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775764549.369350:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffffa0e9d0b40000. 02000000:00000010:0.0:1775764549.369365:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffffa0e9c8130800. 00000100:00000010:0.0:1775764549.369371:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9d0712d80. 00000400:00000010:2.0:1775764549.371382:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c55e5b58. 00000400:00000200:2.0:1775764549.371396:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8200110 00000400:00000010:2.0:1775764549.371399:0:6160:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8200110. 00000100:00000200:2.0:1775764549.371415:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9d0710380 x1862023861243008/t0(0) o900->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:29/10 lens 264/248 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775764549.371429:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9d0710380 x1862023861243008/t0(0) o900->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:29/10 lens 264/248 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775764549.371467:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at ffffa0e9d0759540. 02000000:00000010:2.0:1775764549.371479:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffffa0e9d075b600. 02000000:00000010:2.0:1775764549.371500:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffffa0e9d075a800. 00000100:00000010:2.0:1775764549.371505:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9d0710380. 80000000:00000010:2.0:1775764549.371517:0:6160:0:(fld_cache.c:357:fld_cache_entry_create()) kmalloced '(f_new)': 56 at ffffa0e9c2aa8380. 00000020:00000010:2.0:1775764549.371750:0:6160:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffffa0e9c8130c00. 00000100:00000010:2.0:1775764549.371758:0:6160:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffffa0e9d0710380. 02000000:00000010:2.0:1775764549.371773:0:6160:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffffa0e9d075a800. 00000100:00000010:2.0:1775764549.371873:0:6160:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffffa0e9d0759540. 02000000:00000010:2.0:1775764549.371952:0:6160:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffffa0e9c3400000. 00000400:00000200:2.0:1775764549.371967:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000010:2.0:1775764549.371970:0:6160:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffffa0e9c55e5b58. 00000400:00000010:2.0:1775764549.371976:0:6160:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffffa0e9c8130000. 00000100:00000200:2.0:1775764549.371984:0:6160:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1862023861243136, portal 10 00000400:00000010:2.0:1775764549.371989:0:6160:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffffa0e9d8200110. 00000100:00000200:2.0:1775764549.371993:0:6160:0:(niobuf.c:94:ptl_send_buf()) Sending 448 bytes to portal 23, xid 1862023861243136, offset 0 00000400:00000200:2.0:1775764549.372000:0:6160:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.104@tcp 00000400:00000200:2.0:1775764549.372010:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.372020:0:6160:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.104@tcp local destination 00000400:00000200:2.0:1775764549.372025:0:6160:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775764549.372030:0:6160:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.4@tcp 00000400:00000200:2.0:1775764549.372034:0:6160:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.4@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775764549.372037:0:6160:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.104@tcp 00000400:00000200:2.0:1775764549.372041:0:6160:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.4@tcp NI seq info: [30:30:256:4294967295] 192.168.202.104@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1775764549.372045:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:2.0:1775764549.372057:0:6160:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp:) -> 192.168.202.104@tcp(192.168.202.104@tcp:192.168.202.104@tcp) : PUT try# 0 00000800:00000200:2.0:1775764549.372064:0:6160:0:(socklnd_cb.c:1001:ksocknal_send()) sending 448 bytes in 1 frags to 12345-192.168.202.104@tcp 00000800:00000010:2.0:1775764549.372069:0:6160:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffffa0e9c84db100. 00000800:00000200:2.0:1775764549.372077:0:6160:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffffa0e9d8204cc0] -> 12345-192.168.202.104@tcp (5) 00000800:00000200:2.0:1775764549.372088:0:6160:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.104@tcp ip 192.168.202.104:988 00000800:00000200:2.0:1775764549.372093:0:6160:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffffa0e9c84db100 type 1, nob 544 niov 1 nkiov 1 00000400:00000200:2.0:1775764549.372186:0:6160:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000800:00000200:1.0:1775764549.379450:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1775764549.379463:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffffa0e9c84db100. 00000400:00000200:1.0:1775764549.379474:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.379501:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000400:00000200:1.0:1775764549.379510:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9d8200110 00000400:00000010:1.0:1775764549.379513:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffffa0e9d8200110. 00000100:00000200:1.0:1775764549.379538:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffffa0e9d0710380 x1862023861243136/t0(0) o33->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:23/10 lens 448/66264 e 0 to 0 dl 1775764565 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000400:00000200:1.0:1775764549.465171:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.465181:0:2393:0:(lib-move.c:4815:lnet_parse()) TRACE: 192.168.202.4@tcp(192.168.202.4@tcp) <- 192.168.202.104@tcp : PUT - for me 00000400:00000200:1.0:1775764549.465192:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.465197:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.465204:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.465208:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.202.104@tcp of length 464 into portal 10 MB=0x69d8036d00d00 00000400:00000200:1.0:1775764549.465215:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.104@tcp to cpt 0 00000400:00000200:1.0:1775764549.465221:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.104@tcp of length 464/464 into md 0x341 [32] + 224 00000400:00000200:1.0:1775764549.465243:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1775764549.465248:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.4@tcp->192.168.202.104@tcp: PUT: OK 00000100:00000200:1.0:1775764549.465269:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffffa0e9d0710380 x1862023861243136/t0(0) o33->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:23/10 lens 448/66264 e 0 to 0 dl 1775764565 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000400:00000010:2.0:1775764549.465400:0:6160:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffffa0e9c55e5b58. 00000400:00000200:2.0:1775764549.465412:0:6160:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffffa0e9c8130000 00000400:00000010:2.0:1775764549.465417:0:6160:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 632 at ffffa0e9c8130000. 00000100:00000200:2.0:1775764549.465431:0:6160:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffffa0e9d0710380 x1862023861243136/t0(0) o33->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:23/10 lens 448/66264 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000100:00000200:2.0:1775764549.465443:0:6160:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffffa0e9d0710380 x1862023861243136/t0(0) o33->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:23/10 lens 448/66264 e 0 to 0 dl 1775764565 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000100:00000010:2.0:1775764549.465556:0:6160:0:(client.c:3771:ptlrpc_reqset_free()) kfreed 'set': 144 at ffffa0e9d0759540. 00000002:00000200:2.0:1775764549.465569:0:6160:0:(mdc_request.c:172:mdc_getattr_common()) mode: 40755 00000080:00000010:2.0:1775764549.465600:0:6160:0:(llite_lib.c:751:client_common_fill_super()) kfreed 'op_data': 384 at ffffa0e9d075a400. 00000020:00000010:2.0:1775764549.465633:0:6160:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffffa0e9c9bbeab0. 00000020:00000010:2.0:1775764549.465639:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffffa0e9d075a400. 00000020:00000010:2.0:1775764549.465645:0:6160:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffffa0e9d075b600. 00000020:00000010:2.0:1775764549.465649:0:6160:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffffa0e9c8de3de0. 00000008:00000010:2.0:1775764549.465658:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d0190000. 00020000:00000010:2.0:1775764549.465664:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9c95596c0. 00008000:00000010:2.0:1775764549.465670:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9c8dd8a98. 00000080:00000010:2.0:1775764549.465678:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9c9ac1e70. 00000080:00000010:2.0:1775764549.465682:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9d04aafe8. 00000020:00000010:2.0:1775764549.465685:0:6160:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffffa0e9d075b800. 00000008:00000010:2.0:1775764549.465689:0:6160:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9d016bce0. 00020000:00000010:2.0:1775764549.465693:0:6160:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9d8269d40. 00008000:00000010:2.0:1775764549.465696:0:6160:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9d0499bb0. 00000080:00000010:2.0:1775764549.465701:0:6160:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d018ac30. 00000080:00000010:2.0:1775764549.465706:0:6160:0:(vvp_dev.c:174:vvp_device_alloc()) kmalloced '(vdv)': 72 at ffffa0e9c8de3600. 00000080:00000010:2.0:1775764549.465712:0:6160:0:(vvp_dev.c:182:vvp_device_alloc()) kmalloced '(site)': 576 at ffffa0e9c8130000. 00000020:00000010:2.0:1775764549.465761:0:6160:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at ffffa0e9c9ba8000. 00000020:00000010:2.0:1775764549.465783:0:6160:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffffa0e9c8132000. 00000020:00000010:2.0:1775764549.465788:0:6160:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at ffffa0e9d0759540. 00000008:00000010:2.0:1775764549.465797:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d0191650. 00020000:00000010:2.0:1775764549.465800:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9c9559600. 00008000:00000010:2.0:1775764549.465803:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9c8dd8e20. 00000080:00000010:2.0:1775764549.465806:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9c9ac1bd0. 00000080:00000010:2.0:1775764549.465809:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9d04a8ff8. 00000008:00000010:2.0:1775764549.465813:0:6160:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9d016b340. 00020000:00000010:2.0:1775764549.465816:0:6160:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9d826af40. 00008000:00000010:2.0:1775764549.465818:0:6160:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9d04993f0. 00000080:00000010:2.0:1775764549.465820:0:6160:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d018af00. 00000008:00000010:2.0:1775764549.465825:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d0192178. 00020000:00000010:2.0:1775764549.465827:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9c9559000. 00008000:00000010:2.0:1775764549.465830:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9c8ddb880. 00000080:00000010:2.0:1775764549.465832:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9c9ac1540. 00000080:00000010:2.0:1775764549.465841:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9d04a8000. 00000008:00000010:2.0:1775764549.465857:0:6160:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9d016adc0. 00020000:00000010:2.0:1775764549.465861:0:6160:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9d8268d80. 00008000:00000010:2.0:1775764549.465863:0:6160:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9d0499db0. 00000080:00000010:2.0:1775764549.465866:0:6160:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d018a5a0. 00000008:00000010:2.0:1775764549.465871:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d0196468. 00020000:00000010:2.0:1775764549.465874:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9c9559300. 00008000:00000010:2.0:1775764549.465877:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9c8dd9fc8. 00000080:00000010:2.0:1775764549.465880:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9c9ac1930. 00000080:00000010:2.0:1775764549.465883:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9d04a9488. 00000008:00000010:2.0:1775764549.465886:0:6160:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9d016b8c0. 00020000:00000010:2.0:1775764549.465889:0:6160:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9d826ad00. 00008000:00000010:2.0:1775764549.465891:0:6160:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9d04999b8. 00000080:00000010:2.0:1775764549.465893:0:6160:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d018ae10. 00000008:00000010:2.0:1775764549.465898:0:6160:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffffa0e9d0190b28. 00020000:00000010:2.0:1775764549.465901:0:6160:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffffa0e9c9559540. 00008000:00000010:2.0:1775764549.465903:0:6160:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffffa0e9c8dd98b8. 00000080:00000010:2.0:1775764549.465906:0:6160:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffffa0e9c9ac1498. 00000080:00000010:2.0:1775764549.465909:0:6160:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffffa0e9d04ab230. 00000008:00000010:2.0:1775764549.465912:0:6160:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffffa0e9d016a840. 00020000:00000010:2.0:1775764549.465915:0:6160:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffffa0e9d826a640. 00008000:00000010:2.0:1775764549.465917:0:6160:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffffa0e9d0499c00. 00000080:00000010:2.0:1775764549.465921:0:6160:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffffa0e9d018a2d0. 00020000:00000010:2.0:1775764549.465932:0:6160:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffffa0e9c8de3360. 00020000:00000010:2.0:1775764549.465938:0:6160:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffffa0e9c8de3540. 00020000:00000010:2.0:1775764549.465942:0:6160:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffffa0e9c8de3060. 00020000:00000010:2.0:1775764549.465947:0:6160:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffffa0e9c8de3660. 00000020:00000010:2.0:1775764549.465951:0:6160:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffffa0e9d075b600. 00000020:00000010:2.0:1775764549.465956:0:6160:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffffa0e9c8de3de0. 00000008:00000010:2.0:1775764549.465959:0:6160:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffffa0e9d0190000. 00020000:00000010:2.0:1775764549.465963:0:6160:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffffa0e9c95596c0. 00008000:00000010:2.0:1775764549.465966:0:6160:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffffa0e9c8dd8a98. 00000080:00000010:2.0:1775764549.465975:0:6160:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffffa0e9c9ac1e70. 00000080:00000010:2.0:1775764549.465979:0:6160:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffffa0e9d04aafe8. 00000020:00000010:2.0:1775764549.465982:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffffa0e9d075a400. 00000008:00000010:2.0:1775764549.465985:0:6160:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffffa0e9d016bce0. 00020000:00000010:2.0:1775764549.465991:0:6160:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffffa0e9d8269d40. 00008000:00000010:2.0:1775764549.465995:0:6160:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffffa0e9d0499bb0. 00000080:00000010:2.0:1775764549.465998:0:6160:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffffa0e9d018ac30. 00000020:00000010:2.0:1775764549.466002:0:6160:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffffa0e9d075b800. 00000020:00000010:2.0:1775764549.466004:0:6160:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffffa0e9c9bbeab0. 00000080:00000010:2.0:1775764549.466060:0:6160:0:(super25.c:41:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffffa0e9d110b9c0. 02000000:00000010:2.0:1775764549.466100:0:6160:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 131072 at ffffa0e9c3400000. 02000000:00000010:2.0:1775764549.466141:0:6160:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffffa0e9d075a800. 00000100:00000010:2.0:1775764549.466146:0:6160:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffffa0e9d0710380. 00000080:00000010:2.0:1775764549.466159:0:6160:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffffa0e9c45601a0. 00000080:00000010:2.0:1775764549.466170:0:6160:0:(llite_lib.c:840:client_common_fill_super()) kfreed 'data': 192 at ffffa0e9c4e15300. 00000080:00000010:2.0:1775764549.466174:0:6160:0:(llite_lib.c:841:client_common_fill_super()) kfreed 'osfs': 144 at ffffa0e9c4e15840. 00000080:00000010:2.0:1775764549.466217:0:6160:0:(llite_lib.c:1490:ll_fill_super()) kfreed 'md': 33 at ffffa0e9c576d380. 00000080:00000010:2.0:1775764549.466221:0:6160:0:(llite_lib.c:1492:ll_fill_super()) kfreed 'dt': 33 at ffffa0e9c576da80. 00000080:00000010:2.0:1775764549.466225:0:6160:0:(llite_lib.c:1500:ll_fill_super()) kfreed 'cfg': 88 at ffffa0e9c4ead060. 00000080:02000400:2.0:1775764549.466229:0:6160:0:(llite_lib.c:1506:ll_fill_super()) Mounted lustre-client 00000080:00200008:1.0:1775764549.729240:0:6212:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764549.729257:0:6212:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:1.0:1775764549.729296:0:6212:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764549.729374:0:6212:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 6212 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764549.729386:0:6212:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 6212 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764549.729396:0:6212:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 6212 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764549.729423:0:6212:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job mkdir:6a4854d3-5a79-4a54-82b7-484d046ae78a:6212:1862023861243264:192.168.202.104@tcp:101: 00000100:00100000:1.0:1775764549.729544:0:6212:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c31710c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764549.744937:0:6212:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job mkdir:6a4854d3-5a79-4a54-82b7-484d046ae78a:6212:1862023861243264:192.168.202.104@tcp:101: 00010000:00010000:3.0:1775764549.744968:0:6212:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.744984:0:6212:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.744997:0:6212:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.745011:0:6212:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.745026:0:6212:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.745035:0:6212:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.745042:0:6212:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764549.745055:0:6212:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffffa0e9c8d0f480 x1862023861243264/t0(0) o101->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764565 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764549.745096:0:6212:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.745112:0:6212:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa0e9c9b92c00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1775764549.745121:0:6212:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764549.745179:0:6212:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1775764549.745194:0:6212:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764549.745201:0:6212:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764549.745205:0:6212:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b603e2fcb8 00010000:00010000:3.0:1775764549.745208:0:6212:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764549.745218:0:6212:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764550.059648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764551.090217:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764552.110298:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1775764552.342720:0:6264:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764552.342734:0:6264:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764552.342764:0:6264:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764552.342787:0:6264:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764552.342805:0:6264:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764552.342814:0:6264:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764552.342832:0:6264:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764552.342836:0:6264:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764552.342851:0:6264:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601be7cb8 00010000:00010000:3.0:1775764552.342856:0:6264:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764552.342866:0:6264:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200008:3.0:1775764552.343625:0:6264:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1775764552.343631:0:6264:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764552.343643:0:6264:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764552.343656:0:6264:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1775764552.343669:0:6264:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:3.0:1775764552.343676:0:6264:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764552.343681:0:6264:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764552.343684:0:6264:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764552.343689:0:6264:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601be7cb8 00010000:00010000:3.0:1775764552.343692:0:6264:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1775764552.343700:0:6264:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1775764552.343839:0:6264:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764552.343848:0:6264:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764552.343860:0:6264:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1775764552.343869:0:6264:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1775764552.343875:0:6264:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764552.343879:0:6264:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764552.343881:0:6264:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764552.343885:0:6264:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601be7bb0 00010000:00010000:3.0:1775764552.343887:0:6264:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1775764552.343894:0:6264:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:3.0:1775764552.343901:0:6264:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:3.0:1775764552.343909:0:6264:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:3.0:1775764552.344007:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:3.0:1775764552.344023:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5000 uarg=0000000000000000 00800000:00000004:3.0:1775764552.344031:0:6264:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 282, max_age 284 00000100:00100000:3.0:1775764552.344081:0:6264:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243392:192.168.202.104@tcp:41: 00000100:00100000:3.0:1775764552.345475:0:6264:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d094e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764552.357572:0:6264:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243392:192.168.202.104@tcp:41: 00800000:00000004:0.0:1775764552.357607:0:6264:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317403/353529 objects 1023723/1024000 00000080:00200080:0.0:1775764552.357701:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.357724:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00800000:00000004:0.0:1775764552.357733:0:6264:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age -719, max_age 284 00000100:00100000:0.0:1775764552.357773:0:6264:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243520:192.168.202.104@tcp:41: 00000100:00100000:0.0:1775764552.357982:0:6264:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81c9600 going to sleep for 11 seconds 00010000:00080000:0.0:1775764552.366003:0:6264:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dd880 x1862023861243520/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764568 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764552.366028:0:6264:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243520:192.168.202.104@tcp:41: 00800000:00000004:0.0:1775764552.366064:0:6264:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317443/353529 objects 1023743/1024000 00000080:00200080:0.0:1775764552.366161:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366183:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366191:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366196:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366200:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366203:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366206:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366209:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366213:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366217:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366240:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366245:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366248:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366251:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366255:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366258:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366261:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366264:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366268:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366272:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366276:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366279:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366283:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366287:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366291:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366294:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366298:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366301:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366305:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366308:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366311:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366315:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366320:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366323:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366327:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366338:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366344:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366348:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366351:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366354:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366357:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366360:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366363:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366366:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366369:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366373:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366376:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366379:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366382:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366385:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366388:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366392:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366396:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366400:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366403:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366407:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366410:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366413:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366417:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366420:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366430:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366434:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366437:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366440:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366444:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366447:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366451:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366454:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366457:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366460:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366463:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366466:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366470:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366473:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366477:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366480:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366536:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366542:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366548:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366551:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366555:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366558:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366562:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366566:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366570:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366581:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366586:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366590:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366593:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366596:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366600:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366603:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366606:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366610:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366613:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366616:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366619:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366623:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366626:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366629:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366633:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366635:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366639:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366642:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366645:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366648:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366651:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366655:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366658:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366660:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366670:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366674:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366678:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366681:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366684:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366687:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366692:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366695:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366700:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366703:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366707:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366711:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366715:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366718:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366722:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00800000:00000080:0.0:1775764552.366725:0:6264:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.366730:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.366737:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c5361c00 uarg=0000000000000000 00020000:00000004:0.0:1775764552.366747:0:6264:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age -719, max_age 284 00000100:00100000:0.0:1775764552.366795:0:6264:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243648:192.168.202.104@tcp:13: 00000100:00100000:0.0:1775764552.367056:0:6264:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81c9600 going to sleep for 11 seconds 00010000:00080000:0.0:1775764552.371293:0:6264:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dd880 x1862023861243648/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764568 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764552.371319:0:6264:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243648:192.168.202.104@tcp:13: 00020000:00000004:0.0:1775764552.371385:0:6264:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901369/958279 objects 261776/262144 00000080:00200080:0.0:1775764552.372108:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.372135:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d8308800 uarg=0000000000000000 00020000:00000004:0.0:1775764552.372144:0:6264:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age -719, max_age 284 00000100:00100000:0.0:1775764552.374853:0:6264:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243776:192.168.202.104@tcp:13: 00000100:00100000:0.0:1775764552.375869:0:6264:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81c9600 going to sleep for 11 seconds 00010000:00080000:0.0:1775764552.383754:0:6264:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dd880 x1862023861243776/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764568 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764552.383777:0:6264:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:6264:1862023861243776:192.168.202.104@tcp:13: 00020000:00000004:0.0:1775764552.383830:0:6264:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901369/958279 objects 261776/262144 00000080:00200080:0.0:1775764552.383908:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.383927:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.383934:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.383938:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.383942:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.383945:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.383948:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.383951:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.383954:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.383957:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.383961:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.383964:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.383968:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.383982:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384010:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384015:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384019:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384023:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384027:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384031:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384035:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384038:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384041:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384045:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384049:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384052:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384056:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384060:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384063:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384067:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384070:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384074:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384077:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384080:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384083:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384086:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384089:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384092:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384096:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384106:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384112:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384115:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384119:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384123:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384127:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384131:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384134:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384138:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384141:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384144:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384147:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384150:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384154:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384157:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384160:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384163:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384167:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384170:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384173:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384177:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384180:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384184:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384187:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384190:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384201:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384206:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384209:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384213:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384218:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384222:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384228:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384231:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384234:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384238:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384241:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384245:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384248:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384252:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384256:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384259:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384263:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384267:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384270:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384273:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384277:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384281:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384285:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384288:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384291:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384313:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384320:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384323:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384327:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384331:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384335:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384339:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384343:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384347:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384351:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384354:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384357:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384360:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384364:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384367:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384371:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384375:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384379:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384382:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384386:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384390:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384394:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384397:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384401:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384405:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384429:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384434:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384437:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384441:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384445:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384448:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384453:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384456:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384460:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384464:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200080:0.0:1775764552.384468:0:6264:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9bae8d0 00020000:00000080:0.0:1775764552.384472:0:6264:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d0b41c00 uarg=0000000000000000 00000080:00200008:0.0:1775764552.384549:0:6264:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:2.0:1775764553.132637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764554.156022:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764554.609699:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244032:192.168.202.104@tcp:400: 00000100:00100000:1.0:1775764554.609825:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244288:192.168.202.104@tcp:400: 00000100:00100000:1.0:1775764554.609876:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1775764554.609882:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861243904:192.168.202.104@tcp:400: 00000100:00100000:1.0:1775764554.609918:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244160:192.168.202.104@tcp:400: 00000100:00100000:1.0:1775764554.609939:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244416:192.168.202.104@tcp:400: 00000100:00100000:1.0:1775764554.619414:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861243904:192.168.202.104@tcp:400: 00010000:00080000:1.0:1775764554.619437:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0f800 x1862023861244160/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764570 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764554.619449:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244160:192.168.202.104@tcp:400: 00010000:00080000:1.0:1775764554.619459:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0ea00 x1862023861244416/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764570 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764554.619467:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244416:192.168.202.104@tcp:400: 00000100:00100000:1.0:1775764554.619479:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244032:192.168.202.104@tcp:400: 00010000:00080000:1.0:1775764554.619530:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0f480 x1862023861244288/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764570 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764554.619538:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861244288:192.168.202.104@tcp:400: 00010000:00010000:1.0:1775764555.182283:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764556.206022:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1775764556.213270:0:6457:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:1.0:1775764557.228312:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764558.261238:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764559.280769:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764559.726224:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861244544:192.168.202.104@tcp:400: 00000100:00100000:3.0:1775764559.726391:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861244800:192.168.202.104@tcp:400: 00000100:00100000:3.0:1775764559.726430:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861245056:192.168.202.104@tcp:400: 00000100:00100000:3.0:1775764559.726482:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764559.726512:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861244672:192.168.202.104@tcp:400: 00000100:00100000:3.0:1775764559.726540:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861244928:192.168.202.104@tcp:400: 00000100:00100000:3.0:1775764559.752927:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861244672:192.168.202.104@tcp:400: 00010000:00080000:3.0:1775764559.752957:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542d80 x1862023861244928/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764575 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764559.752969:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861244928:192.168.202.104@tcp:400: 00000100:00100000:3.0:1775764559.752980:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861244544:192.168.202.104@tcp:400: 00010000:00080000:3.0:1775764559.752994:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540000 x1862023861244800/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764575 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764559.753003:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861244800:192.168.202.104@tcp:400: 00010000:00080000:3.0:1775764559.753015:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540380 x1862023861245056/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764575 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764559.753024:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861245056:192.168.202.104@tcp:400: 00010000:00010000:0.0:1775764560.304630:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764561.132380:0:2394:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.104@tcp (source: 12345-192.168.202.104@tcp) 00000100:00100000:0.0:1775764561.132544:0:2394:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.104@tcp (source: 12345-192.168.202.104@tcp) 00000100:00100000:1.0:1775764561.147260:0:6161:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862023885418368 00000100:00100000:1.0:1775764561.147305:0:6161:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862023885418368 00000100:00100000:1.0:1775764561.147702:0:6161:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862023885418496 00000100:00100000:1.0:1775764561.147747:0:6161:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862023885418496 00000100:00100000:1.0:1775764561.147769:0:6161:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.104@tcp, seq: 0 00000100:00100000:1.0:1775764561.147779:0:6161:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffffa0e9d09dd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+4:10285:x1862023885418368:12345-192.168.202.104@tcp:104: 00010000:00010000:1.0:1775764561.147798:0:6161:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764561.147900:0:6161:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764561.147905:0:6161:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffffa0e9d09dd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+4:10285:x1862023885418368:12345-192.168.202.104@tcp:104: Request processed in 129us (15547us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764561.147913:0:6161:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.104@tcp, seq: 0 00000100:00100000:1.0:1775764561.148146:0:6161:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.104@tcp, seq: 1 00000100:00100000:1.0:1775764561.148155:0:6161:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffffa0e9d09df800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:10285:x1862023885418496:12345-192.168.202.104@tcp:104: 00010000:00010000:1.0:1775764561.148164:0:6161:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764561.148295:0:6161:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764561.148301:0:6161:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffffa0e9d09df800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:10285:x1862023885418496:12345-192.168.202.104@tcp:104: Request processed in 148us (15758us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764561.148307:0:6161:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.104@tcp, seq: 1 00010000:00010000:3.0:1775764561.151186:0:6171:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764561.151203:0:6171:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764561.151215:0:6171:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0ea02052a00 already unused, calling callback (ffffffffc112fc20) 10000000:00010000:3.0:1775764561.151220:0:6171:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764561.151259:0:6171:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764561.151267:0:6171:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764561.151273:0:6171:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1775764561.153238:0:6172:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764561.153467:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764561.153480:0:6172:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764561.154734:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764561.154750:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764561.156412:0:6172:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0ea02052600 already unused, calling callback (ffffffffc112fc20) 10000000:00010000:2.0:1775764561.156429:0:6172:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764561.156449:0:6172:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1775764561.156482:0:6172:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764561.156500:0:6172:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1775764561.156706:0:6172:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764561.156706:0:6171:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764561.156721:0:6172:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764561.156784:0:6172:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764561.156792:0:6172:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0ea02052600/0x64300da17553cb70 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eab1 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764561.156819:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764561.156821:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764561.156823:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764561.156874:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764561.156912:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861245312:192.168.202.104@tcp:103: 00010000:00010000:3.0:1775764561.157212:0:6171:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764561.157261:0:6171:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764561.157269:0:6171:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0ea02052a00/0x64300da17553cb77 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eab8 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764561.157299:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764561.157301:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764561.157305:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764561.157346:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764561.157371:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861245184:192.168.202.104@tcp:103: 00000100:00100000:2.0:1775764561.174228:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861245312:192.168.202.104@tcp:103: 00000100:00100000:3.0:1775764561.174797:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861245184:192.168.202.104@tcp:103: 00010000:00010000:0.0:1775764561.326560:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1775764562.294385:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:0.0:1775764562.294393:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-client-ffffa0e9c80cb000 from 48 10000000:01000000:0.0:1775764562.294399:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1775764562.294451:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764562.294463:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764562.294511:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764562.294537:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861245440:192.168.202.104@tcp:101: 00000100:00100000:0.0:1775764562.294886:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81b0000 going to sleep for 11 seconds 00000100:00100000:0.0:1775764562.302712:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861245440:192.168.202.104@tcp:101: 00010000:00010000:0.0:1775764562.302755:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb1a expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764562.302771:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76eb1a expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764562.302851:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861245568:192.168.202.104@tcp:501: 00000100:00100000:0.0:1775764562.302989:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81b0000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764562.329483:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861245568:192.168.202.104@tcp:501: 00000100:00100000:1.0:1775764562.329580:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861245696:192.168.202.104@tcp:503: 00010000:00010000:3.0:1775764562.347612:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764562.356990:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c980f300 going to sleep for 11 seconds 00000100:00100000:1.0:1775764562.365782:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861245696:192.168.202.104@tcp:503: 00000100:00100000:1.0:1775764562.366040:0:6664:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6664:1862023861245824:192.168.202.104@tcp:502: 00000100:00100000:1.0:1775764562.366281:0:6664:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c980f240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764562.385717:0:6664:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6664:1862023861245824:192.168.202.104@tcp:502: 00000020:01000000:2.0:1775764562.385783:0:6664:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1775764562.385799:0:6664:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1775764562.385802:0:6664:0:(obd_config.c:1559:class_process_config()) marker 40 (0x1) lustre sys.jobid_var 00000020:00000080:2.0:1775764562.385811:0:6664:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 00000100:00100000:2.0:1775764562.385877:0:6664:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6664:1862023861245952:192.168.202.104@tcp:502:llog_process_th.0 00000100:00100000:2.0:1775764562.386007:0:6664:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d06106c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764562.421650:0:6664:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6664:1862023861245952:192.168.202.104@tcp:502:llog_process_th.0 00000020:01000000:1.0:1775764562.421713:0:6664:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1775764562.421735:0:6664:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764562.421738:0:6664:0:(obd_config.c:1559:class_process_config()) marker 40 (0x2) lustre sys.jobid_var 00000040:00080000:1.0:1775764562.421742:0:6664:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 52 count 51 00000020:01000000:3.0:1775764562.424736:0:6174:0:(obd_config.c:2143:class_config_parse_llog()) Processed log lustre-client gen 48-51 (rc=0) 10000000:01000000:3.0:1775764562.424781:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1775764562.424792:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb1a expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764562.424833:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb1a expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764562.424848:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:3.0:1775764562.424851:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-sptlrpc-ffffa0e9d0958000 from 1 10000000:01000000:3.0:1775764562.424857:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:3.0:1775764562.424913:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764562.424921:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764562.424934:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764562.424988:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861246080:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1775764562.425115:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81c9f00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764562.429112:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861246080:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1775764562.429151:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb83 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764562.429165:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76eb83 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764562.429264:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861246208:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1775764562.429353:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81c9f00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764562.433078:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861246208:192.168.202.104@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1775764562.433182:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1775764562.433199:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb83 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764562.433216:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb83 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764563.371669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764564.397355:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764564.698404:0:2393:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.104@tcp (source: 12345-192.168.202.104@tcp) 00000100:00100000:0.0:1775764564.698540:0:6161:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862023885428480 00000100:00100000:0.0:1775764564.698553:0:6161:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862023885428480 00000100:00100000:0.0:1775764564.698646:0:6161:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.104@tcp, seq: 2 00000100:00100000:0.0:1775764564.698658:0:6161:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffffa0e9c8d0d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:10347:x1862023885428480:12345-192.168.202.104@tcp:104: 00010000:00010000:0.0:1775764564.698680:0:6161:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764564.698772:0:6161:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764564.698779:0:6161:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffffa0e9c8d0d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:10347:x1862023885428480:12345-192.168.202.104@tcp:104: Request processed in 124us (386us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764564.698786:0:6161:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.104@tcp, seq: 2 00010000:00010000:3.0:1775764564.699060:0:6172:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764564.699068:0:6172:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764564.699076:0:6172:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0e9c84a3a00 already unused, calling callback (ffffffffc112fc20) 10000000:00010000:3.0:1775764564.699079:0:6172:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764564.699088:0:6172:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764564.699094:0:6172:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764564.699099:0:6172:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1775764564.699149:0:6172:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764564.699156:0:6172:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764564.699178:0:6172:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764564.699194:0:6172:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbaf lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eac6 expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764564.699225:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764564.699227:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764564.699229:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764564.699253:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861246336:192.168.202.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764564.703784:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861246336:192.168.202.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764564.844360:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246464:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764564.844465:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246720:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764564.844528:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1775764564.844536:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246592:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764564.844560:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246848:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764564.861858:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0f800 x1862023861246592/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764580 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764564.861885:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246592:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764564.861899:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861246848/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764580 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764564.861908:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246848:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764564.861920:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246464:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764564.862111:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0d500 x1862023861246720/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764580 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764564.862144:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861246720:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764565.424825:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764566.443802:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1775764566.509178:0:6174:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1775764566.509191:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log params-ffffa0e9c80cb000 from 4 10000000:01000000:3.0:1775764566.509198:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1775764566.509273:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764566.509290:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764566.509307:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764566.509342:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861246976:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1775764566.511273:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81c9cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764566.536188:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861246976:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1775764566.536230:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764566.536270:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764566.536401:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861247104:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1775764566.536814:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81c9cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764566.547226:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861247104:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764566.547408:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861247232:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1775764566.547588:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d0610a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764566.550821:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861247232:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1775764566.551453:0:6729:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6729:1862023861247360:192.168.202.104@tcp:502:llog_process_th.0 00000100:00100000:2.0:1775764566.551617:0:6729:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d0610240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764566.557826:0:6729:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6729:1862023861247360:192.168.202.104@tcp:502:llog_process_th.0 00000020:01000000:2.0:1775764566.557920:0:6729:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1775764566.557936:0:6729:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1775764566.557940:0:6729:0:(obd_config.c:1559:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:2.0:1775764566.557950:0:6729:0:(obd_config.c:2003:class_config_llog_handler()) cmd ce032, instance name: general-ffffa0e9c80cb000 00000020:00000080:2.0:1775764566.557954:0:6729:0:(obd_config.c:1468:class_process_config()) processing cmd: ce032 00000100:00100000:2.0:1775764566.558101:0:6729:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6729:1862023861247488:192.168.202.104@tcp:502:llog_process_th.0 00000100:00100000:2.0:1775764566.558220:0:6729:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d0610240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764566.580918:0:6729:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:2bf32c54-7e4b-4438-8419-d956344e0e00:6729:1862023861247488:192.168.202.104@tcp:502:llog_process_th.0 00000020:01000000:0.0:1775764566.581000:0:6729:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1775764566.581015:0:6729:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1775764566.581018:0:6729:0:(obd_config.c:1559:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:0.0:1775764566.581028:0:6729:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:3.0:1775764566.581158:0:6174:0:(obd_config.c:2143:class_config_parse_llog()) Processed log params gen 4-6 (rc=0) 10000000:01000000:3.0:1775764566.581200:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1775764566.581218:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764566.581235:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764567.467851:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764568.492557:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764569.517872:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764569.964140:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861247744:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764569.964296:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248000:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764569.964334:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1775764569.964340:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861247616:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764569.964373:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861247872:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764569.964398:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248128:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764569.967155:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861247616:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764569.970868:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542300 x1862023861248128/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764585 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764569.970894:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248128:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764569.970911:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0541c00 x1862023861248000/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764585 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764569.970921:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248000:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764569.975284:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542d80 x1862023861247872/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764585 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764569.975305:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861247872:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764569.975321:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861247744:192.168.202.104@tcp:400:kworker.0 00010000:00010000:3.0:1775764570.539739:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764571.563944:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764572.587802:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764573.611661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764574.641685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764575.083942:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1775764575.083949:0:37:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffffa0e9c80cb000: disconnect after 23s idle 00000100:00080000:3.0:1775764575.091238:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1775764575.091244:0:37:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffffa0e9c80cb000: disconnect after 23s idle 00000100:00100000:3.0:1775764575.091329:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861248256:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764575.091615:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861248640:192.168.202.104@tcp:9:kworker.0 00000100:00100000:0.0:1775764575.091731:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861248768:192.168.202.104@tcp:9:kworker.0 00000100:00100000:3.0:1775764575.092154:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248512:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764575.092190:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764575.092199:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248384:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764575.100840:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542300 x1862023861248640/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764585 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764575.100865:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9d0542300 x1862023861248640/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764585 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764575.100874:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1775764575.100879:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861248640:192.168.202.104@tcp:9:kworker.0 00010000:00080000:0.0:1775764575.100896:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540000 x1862023861248768/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764585 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764575.100906:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9d0540000 x1862023861248768/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764585 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764575.100913:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1775764575.100917:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861248768:192.168.202.104@tcp:9:kworker.0 00000100:00100000:3.0:1775764575.101639:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248384:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764575.101666:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0541c00 x1862023861248512/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764591 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764575.101676:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861248512:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764575.102077:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861248256:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764575.667386:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764576.691174:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764577.370159:0:7013:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg204-client.virtnet: executing check_logdir /tmp/testlogs/ 00010000:00010000:1.0:1775764577.713765:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764578.733744:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764579.764346:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764580.206024:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249024:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764580.206152:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1775764580.206159:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861248896:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764580.208760:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249152:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764580.228528:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861248896:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764580.231736:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542d80 x1862023861249152/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764596 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764580.231758:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249152:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764580.231774:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249024:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764580.779739:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764581.805124:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764582.828825:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764583.768431:0:7276:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg204-client.virtnet: executing yml_node 00010000:00010000:0.0:1775764583.853042:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764584.883943:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764585.325421:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861249280:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764585.325565:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249536:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764585.325602:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764585.325610:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249408:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764585.336284:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249408:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764585.336332:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861249280:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764585.336353:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0710e00 x1862023861249536/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764601 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764585.336367:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249536:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764585.902782:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764586.923864:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764587.951049:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764588.486129:0:7730:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: Client: 2.17.51.76 00010000:00010000:0.0:1775764588.971979:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764589.999756:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764590.444089:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249792:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764590.444092:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861249664:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764590.447415:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861249920:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764590.456675:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861249664:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764590.456710:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0543800 x1862023861249920/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764606 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764590.456727:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861249920:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764590.471228:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861249792:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764591.026767:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764591.361772:0:7887:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: MDS: 2.17.51.76 00010000:00010000:0.0:1775764592.046289:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764593.067609:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764594.093640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764594.338482:0:8043:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: OSS: 2.17.51.76 00010000:00010000:2.0:1775764595.121189:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764595.574602:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861250048:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764595.574716:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861250304:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764595.574763:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764595.574770:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861250176:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764595.581445:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861250048:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764595.586851:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861250176:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764595.586877:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713b80 x1862023861250304/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764611 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764595.586890:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861250304:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764596.145524:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764596.712044:0:8189:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: -----============= acceptance-small: insanity ============----- Thu Apr 9 15:56:34 EDT 2026 00010000:00010000:2.0:1775764597.164451:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764598.188310:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764599.214564:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764600.235921:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764600.683863:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861250560:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764600.683992:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764600.683999:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861250432:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764600.684223:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861250688:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764600.689726:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861250432:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764600.689752:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861250688/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764616 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764600.689768:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861250688:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764600.689781:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861250560:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764601.265750:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764602.284962:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764603.309701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764604.333515:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764605.356449:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764605.803785:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861250944:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764605.803884:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1775764605.803891:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861250816:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764605.803932:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861251072:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764605.813719:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861250816:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764605.813761:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dc380 x1862023861251072/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764621 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764605.813775:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861251072:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764605.813787:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861250944:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764606.380825:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764607.404018:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764608.428016:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764609.455879:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764610.475807:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764610.924305:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251328:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764610.924421:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764610.924429:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861251200:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764610.924458:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251456:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764610.933565:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861251200:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764610.933600:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dfb80 x1862023861251456/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764626 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764610.933620:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251456:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764610.933663:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251328:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764611.511072:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764612.528121:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764613.547786:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764614.575310:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764615.595798:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764616.046689:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861251584:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764616.046784:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251840:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764616.046819:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764616.046825:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251712:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764616.050387:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251712:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764616.050406:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0d500 x1862023861251840/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764632 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764616.050430:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861251840:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764616.054718:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861251584:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764616.620271:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764617.650328:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764618.669235:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764619.116985:0:8721:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: excepting tests: 00010000:00010000:0.0:1775764619.691663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764620.716355:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764621.167201:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861251968:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764621.171402:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252224:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764621.171533:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764621.171542:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252096:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764621.177589:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252096:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764621.177616:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dca80 x1862023861252224/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764637 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764621.177629:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252224:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764621.178266:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861251968:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764621.739649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764622.268458:0:8942:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: === insanity: start setup 15:57:00 (1775764620) === 00000080:00200008:0.0:1775764622.633691:0:8206:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764622.633703:0:8206:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764622.633732:0:8206:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764622.633753:0:8206:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:0.0:1775764622.633770:0:8206:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:0.0:1775764622.633791:0:8206:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764622.633801:0:8206:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764622.633804:0:8206:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:0.0:1775764622.633846:0:8206:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601d57cb8 00010000:00010000:0.0:1775764622.633851:0:8206:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:0.0:1775764622.633859:0:8206:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200008:3.0:1775764622.657890:0:8206:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764622.657900:0:8206:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764622.657920:0:8206:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764622.657937:0:8206:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1775764622.657951:0:8206:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:3.0:1775764622.657960:0:8206:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764622.657969:0:8206:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764622.657973:0:8206:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764622.657980:0:8206:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601d57cb8 00010000:00010000:3.0:1775764622.657985:0:8206:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1775764622.657993:0:8206:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200008:3.0:1775764622.678632:0:8206:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764622.678645:0:8206:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764622.678674:0:8206:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764622.678695:0:8206:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1775764622.678719:0:8206:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:3.0:1775764622.678742:0:8206:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764622.678751:0:8206:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764622.678754:0:8206:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764622.678762:0:8206:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601d57cb8 00010000:00010000:3.0:1775764622.678766:0:8206:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1775764622.678776:0:8206:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200008:3.0:1775764622.698591:0:8206:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764622.698605:0:8206:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764622.698641:0:8206:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764622.698669:0:8206:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:3.0:1775764622.698688:0:8206:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:3.0:1775764622.698697:0:8206:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764622.698706:0:8206:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764622.698709:0:8206:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764622.698718:0:8206:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601d57cb8 00010000:00010000:3.0:1775764622.698731:0:8206:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:3.0:1775764622.698740:0:8206:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:0.0:1775764622.767360:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764623.788000:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764624.814384:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764625.845374:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764626.287118:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861252352:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764626.287255:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861252608:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764626.287294:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764626.287300:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861252480:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764626.289684:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861252352:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764626.312403:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861252480:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764626.312431:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540e00 x1862023861252608/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764642 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764626.312444:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861252608:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764626.859810:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764627.883877:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764628.277435:0:9193:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg204-client.virtnet: executing check_config_client /mnt/lustre 00010000:00010000:2.0:1775764628.912584:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764629.931846:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764630.956401:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764631.403742:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252864:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764631.403853:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1775764631.403889:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861252736:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764631.403919:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252992:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764631.417070:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861252736:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764631.439742:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861252992/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764647 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764631.439772:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252992:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764631.439790:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861252864:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764631.984735:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764633.003805:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764634.027681:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764635.051892:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764636.076476:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764636.525662:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861253120:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764636.525814:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861253376:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764636.526107:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764636.526115:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861253248:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764636.530290:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861253248:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764636.530313:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0e680 x1862023861253376/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764652 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764636.530325:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861253376:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764636.533317:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861253120:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764637.100413:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764638.124227:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764639.147753:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764640.172602:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764641.196560:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764641.650205:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861253504:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764641.650379:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861253760:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764641.650422:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764641.650429:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861253632:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764641.659580:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861253504:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764641.659620:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09df480 x1862023861253760/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764657 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764641.659672:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861253760:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764641.659727:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861253632:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764642.219606:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764643.248254:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764644.270898:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764645.294334:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764646.315963:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764646.764432:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861253888:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764646.772466:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764646.773623:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861254016:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764646.773815:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861253888:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764646.775855:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764646.775901:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861254144:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764646.789586:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861254016:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764646.793162:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0710a80 x1862023861254144/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764662 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764646.793190:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861254144:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764647.340988:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764648.364377:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764649.388978:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764650.414981:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764651.438155:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764651.886416:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861254272:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764651.887860:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764651.887888:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861254400:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764651.893044:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861254400:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764651.893062:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861254272:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764651.894463:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861254528:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764651.900274:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0f800 x1862023861254528/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764667 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764651.900298:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861254528:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764652.461546:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1775764652.884298:0:9934:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:1.0:1775764653.485312:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764654.508121:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764655.531835:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764656.555979:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764656.845987:0:2393:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.104@tcp (source: 12345-192.168.202.104@tcp) 00000100:00100000:0.0:1775764656.846127:0:6162:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862023885474560 00000100:00100000:0.0:1775764656.846152:0:6162:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862023885474560 00000100:00100000:0.0:1775764656.846233:0:6162:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.104@tcp, seq: 3 00000100:00100000:0.0:1775764656.846246:0:6162:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffffa0e9c8d0f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2bf32c54-7e4b-4438-8419-d956344e0e00+3:13210:x1862023885474560:12345-192.168.202.104@tcp:104: 00010000:00010000:0.0:1775764656.846271:0:6162:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764656.846590:0:6162:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764656.846599:0:6162:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffffa0e9c8d0f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2bf32c54-7e4b-4438-8419-d956344e0e00+3:13210:x1862023885474560:12345-192.168.202.104@tcp:104: Request processed in 356us (629us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764656.846608:0:6162:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.104@tcp, seq: 3 00010000:00010000:2.0:1775764656.848065:0:6171:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764656.848087:0:6171:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764656.848102:0:6171:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0e9c84a3a00 already unused, calling callback (ffffffffc112fc20) 10000000:00010000:2.0:1775764656.848108:0:6171:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764656.848120:0:6171:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1775764656.848128:0:6171:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764656.848135:0:6171:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1775764656.848214:0:6171:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764656.848221:0:6171:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764656.848257:0:6171:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764656.848263:0:6171:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3a00/0x64300da17553cbf5 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76eb91 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764656.848294:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764656.848297:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764656.848299:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764656.848326:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861254656:192.168.202.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764656.870530:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861254656:192.168.202.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764657.006700:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861254784:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764657.006837:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764657.006845:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861254912:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764657.029815:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0ea00 x1862023861254912/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764673 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764657.029841:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861254912:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764657.029879:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861254784:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764657.594667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764658.604652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1775764658.605282:0:6174:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:0.0:1775764658.605288:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log params-ffffa0e9c80cb000 from 7 10000000:01000000:0.0:1775764658.605294:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:0.0:1775764658.605365:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764658.605379:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764658.605392:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764658.605431:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861255040:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764658.605655:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c4f08240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764658.611159:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861255040:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1775764658.611204:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ec0f expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764658.611221:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76ec0f expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764658.611327:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861255168:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1775764658.611478:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c4f08240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764658.625757:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861255168:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1775764658.625877:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861255296:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000100:00100000:0.0:1775764658.626018:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c4f08240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764658.632469:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861255296:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1775764658.640389:0:10211:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:3.0:1775764658.641064:0:6174:0:(obd_config.c:2143:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:3.0:1775764658.641128:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1775764658.641142:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ec0f expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764658.641158:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ec0f expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764659.630687:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764660.651857:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764661.675971:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764662.126273:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861255424:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764662.126432:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861255680:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764662.126481:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764662.126518:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861255552:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764662.134796:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861255552:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764662.134954:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861255424:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764662.134978:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0541180 x1862023861255680/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764678 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764662.134992:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861255680:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764662.706984:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764663.723935:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764664.747882:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764665.771648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764666.805416:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764667.250215:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861255936:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764667.250364:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764667.250371:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861255808:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764667.250402:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861256064:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764667.256676:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861255808:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764667.256718:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711f80 x1862023861256064/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764683 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764667.256733:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861256064:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764667.256748:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861255936:192.168.202.104@tcp:400:kworker.0 00010000:00010000:3.0:1775764667.820127:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764668.310464:0:10668:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: === insanity: finish setup 15:57:46 (1775764666) === 00000080:00200000:1.0:1775764668.722254:0:10682:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:1.0:1775764668.722293:0:10682:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764668.722314:0:10682:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764668.722331:0:10682:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764668.722340:0:10682:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764668.722355:0:10682:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:1.0:1775764668.722358:0:10682:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:1.0:1775764668.722367:0:10682:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601c7fb58 00010000:00010000:1.0:1775764668.722370:0:10682:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764668.722379:0:10682:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764668.722401:0:10682:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764668.722416:0:10682:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d0.insanity, dir=[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags=8 00000080:00200000:1.0:1775764668.722420:0:10682:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d0.insanity, dir=[0x200000007:0x1:0x0](ffffa0e9d110ba88), intent=getattr 00000002:00010000:1.0:1775764668.722434:0:10682:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d0.insanity,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764668.722480:0:10682:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764668.724134:0:10682:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764668.724158:0:10682:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764668.724214:0:10682:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job rm:6a4854d3-5a79-4a54-82b7-484d046ae78a:10682:1862023861256192:192.168.202.104@tcp:101:rm.0 00000100:00100000:1.0:1775764668.724398:0:10682:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81f6480 going to sleep for 11 seconds 00000100:00100000:3.0:1775764668.743324:0:10682:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job rm:6a4854d3-5a79-4a54-82b7-484d046ae78a:10682:1862023861256192:192.168.202.104@tcp:101:rm.0 00010000:00010000:3.0:1775764668.743358:0:10682:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743373:0:10682:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743381:0:10682:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743389:0:10682:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743398:0:10682:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743431:0:10682:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0e9c164f600 already unused, calling callback (ffffffffc19012e0) 00010000:00010000:3.0:1775764668.743438:0:10682:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743446:0:10682:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743454:0:10682:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743463:0:10682:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764668.743470:0:10682:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764668.743477:0:10682:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164f600/0x64300da17553cc03 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 10682 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764668.743558:0:10682:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffffa0e9c8d0e680 x1862023861256192/t0(0) o101->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764684 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764668.743579:0:10682:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1775764668.743622:0:10682:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1775764668.743636:0:10682:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00010000:3.0:1775764668.743646:0:10682:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601c7f930 00010000:00010000:3.0:1775764668.743651:0:10682:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1775764668.743671:0:10682:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1775764668.743895:0:10682:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764668.743915:0:10682:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764668.743928:0:10682:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764668.743946:0:10682:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1775764668.743954:0:10682:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764668.743961:0:10682:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764668.743966:0:10682:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:2.0:1775764668.743972:0:10682:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601c7fc10 00010000:00010000:2.0:1775764668.743975:0:10682:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764668.743983:0:10682:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764668.743994:0:10682:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764668.744008:0:10682:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d0.insanity, flags=0 00000080:00200000:3.0:1775764668.762674:0:10683:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764668.762699:0:10683:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764668.762718:0:10683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1775764668.762737:0:10683:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1775764668.762759:0:10683:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764668.762768:0:10683:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764668.762773:0:10683:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764668.762781:0:10683:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601ce7e18 00010000:00010000:3.0:1775764668.762785:0:10683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1775764668.762793:0:10683:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:3.0:1775764668.762809:0:10683:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 101 00000080:00200000:3.0:1775764668.762820:0:10683:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764668.762826:0:10683:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764668.762833:0:10683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:3.0:1775764668.762851:0:10683:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:3.0:1775764668.762858:0:10683:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764668.762862:0:10683:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764668.762864:0:10683:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764668.762868:0:10683:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601ce7be0 00010000:00010000:3.0:1775764668.762870:0:10683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:3.0:1775764668.762876:0:10683:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1775764668.762883:0:10683:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764668.762891:0:10683:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d0.insanity, flags=1536 00000080:00200000:3.0:1775764668.762897:0:10683:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764668.762904:0:10683:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764668.762909:0:10683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1775764668.762916:0:10683:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1775764668.762922:0:10683:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764668.762926:0:10683:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764668.762928:0:10683:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cbe0 00000080:00010000:3.0:1775764668.762932:0:10683:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cbe0 from it ffffb6b601ce7dc8 00010000:00010000:3.0:1775764668.762941:0:10683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1775764668.762948:0:10683:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1775764668.762956:0:10683:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 3 00000080:00200000:3.0:1775764668.762963:0:10683:0:(namei.c:2216:do_mkdir()) VFS Op:name=d0.insanity, dir=[0x200000007:0x1:0x0](ffffa0e9d110ba88) 00800000:00000004:3.0:1775764668.762994:0:10683:0:(obd_class.h:1058:obd_statfs_async()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 285, max_age 0 00000100:00100000:3.0:1775764668.763076:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764668.763104:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861256320:192.168.202.104@tcp:41:mkdir.0 00000100:00100000:3.0:1775764668.763265:0:10683:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job mkdir:6a4854d3-5a79-4a54-82b7-484d046ae78a:10683:1862023861256448:192.168.202.104@tcp:700:mkdir.0 00000100:00100000:3.0:1775764668.763290:0:10683:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d8480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764668.770099:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861256320:192.168.202.104@tcp:41:mkdir.0 00000100:00100000:3.0:1775764668.770211:0:10683:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job mkdir:6a4854d3-5a79-4a54-82b7-484d046ae78a:10683:1862023861256448:192.168.202.104@tcp:700:mkdir.0 00010000:00010000:3.0:1775764668.770246:0:10683:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1775764668.770317:0:10683:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1775764668.770325:0:10683:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764668.770328:0:10683:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c164ea00/0x64300da17553cbe0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x39b1144a2f76eaf0 expref: -99 pid: 6212 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000020:00100000:3.0:1775764668.770358:0:10683:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa0e9c80cb000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764668.770366:0:10683:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0e9d0540a80 x1862023861256704/t0(0) o36->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 560/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764668.770408:0:10683:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job mkdir:6a4854d3-5a79-4a54-82b7-484d046ae78a:10683:1862023861256704:192.168.202.104@tcp:36:mkdir.0 00000100:00100000:3.0:1775764668.770519:0:10683:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d8480 going to sleep for 11 seconds 00000100:00100000:3.0:1775764668.773860:0:10683:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job mkdir:6a4854d3-5a79-4a54-82b7-484d046ae78a:10683:1862023861256704:192.168.202.104@tcp:36:mkdir.0 00000080:00200000:3.0:1775764668.773942:0:10683:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000402:0x1:0x0](ffffa0e9d115ec08) 00000080:00200000:3.0:1775764668.773951:0:10683:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000402:0x1:0x0], updating i_size 4096 00010000:00010000:2.0:1775764668.845321:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764669.881672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764670.858226:0:10821:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == insanity test 0: Fail all nodes, independently ======== 15:57:48 (1775764668) 00010000:00010000:1.0:1775764670.896998:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764671.915725:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764672.365436:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861256832:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764672.368274:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764672.368297:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861256960:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764672.374678:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764672.374735:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861257088:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764672.393546:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09ddf80 x1862023861257088/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764688 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764672.393567:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861257088:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764672.393691:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861256960:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764672.393701:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861256832:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764672.940768:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764673.964265:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764674.988010:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764676.014798:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764677.041776:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764677.489105:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861257216:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764677.489293:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861257472:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764677.489333:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764677.489340:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861257344:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764677.505113:0:2403:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-MDT0000-mdc-ffffa0e9c80cb000 of lustre-MDT0000_UUID@192.168.202.104@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1775764677.505121:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1775764677.505126:0:2403:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa0e9c80cb000: Connection to lustre-MDT0000 (at 192.168.202.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1775764677.531009:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764677.531027:0:2403:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 281 00000100:00080000:3.0:1775764677.531044:0:2403:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764677.531094:0:2403:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764677.531133:0:2403:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861257600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764677.531160:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861257344:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764677.531193:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0710a80 x1862023861257472/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764693 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764677.531218:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861257472:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764677.532180:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861257600:192.168.202.104@tcp:38:ptlrpcd_01_00.0 00000100:00080000:0.0:1775764677.539855:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764677.539868:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:0.0:1775764677.539874:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861257600:192.168.202.104@tcp:38:ptlrpcd_01_00.0 00010000:00010000:2.0:1775764678.060874:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764679.094221:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764680.108377:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764681.132016:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764682.155902:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764682.604461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764682.604470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764682.604476:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764682.604509:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 411 00000100:00080000:1.0:1775764682.604524:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1775764682.604532:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775764682.604544:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764682.604564:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8d0df80 x1862023861257856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764682.604744:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861257728:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764682.604858:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764682.604863:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861257984:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764682.604980:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861257856:192.168.202.104@tcp:38:kworker.0 00010000:00080000:0.0:1775764682.615239:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9c8d0f480 x1862023861257984/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764698 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764682.615274:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861257984:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764682.654572:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764682.654592:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764682.654599:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861257856:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775764683.181215:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764684.206706:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764685.227900:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764686.251894:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764687.279636:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764687.724531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764687.724538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764687.724543:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764687.724557:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 416 00000100:00080000:3.0:1775764687.724579:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1775764687.724587:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764687.724598:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764687.724620:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861258240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764687.724684:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861258112:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764687.724836:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764687.724842:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861258368:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764687.725926:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861258240:192.168.202.104@tcp:38:kworker.0 00010000:00080000:3.0:1775764687.744239:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0543100 x1862023861258368/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764703 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764687.744263:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861258368:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764687.768369:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764687.768390:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764687.768396:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861258240:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775764688.300806:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764689.325614:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764690.348817:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764691.372329:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764692.401732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764692.843916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764692.843925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764692.843931:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764692.843947:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 421 00000100:00080000:0.0:1775764692.843968:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1775764692.843990:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764692.844008:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764692.844033:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861258624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764692.844098:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861258752:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764692.844157:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861258624:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775764692.844261:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861258496:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764692.896257:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764692.896282:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764692.896290:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861258624:192.168.202.104@tcp:38:kworker.0 00010000:00080000:1.0:1775764692.897287:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d09df480 x1862023861258752/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764708 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764692.897354:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861258752:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764693.423602:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1775764693.740055:0:2403:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764677/real 1775764677] req@ffffa0e9d0713100 x1862023861257216/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764693 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764693.790380:0:2403:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa0e9d0713100 x1862023861257216/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764693 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764693.791567:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1775764693.791575:0:2403:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.202.104@tcp: Connection to MGS (at 192.168.202.104@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1775764693.803205:0:2403:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.202.104@tcp for MGC192.168.202.104@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1775764693.803211:0:2403:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1775764693.803221:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9d0713100 x1862023861257216/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764693 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764693.803236:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9c8d0d500 x1862023861257728/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764698 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764693.803340:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9d0541f80 x1862023861258112/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764703 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764693.803369:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9d09de680 x1862023861258496/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764708 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1775764693.803379:0:2403:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1775764693.803382:0:2403:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1775764693.803383:0:2403:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1775764693.803385:0:2403:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:3.0:1775764693.803429:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861257216:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764693.806466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764693.806478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775764693.806540:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764693.806552:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 281 00000100:00080000:0.0:1775764693.806573:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775764693.806651:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861258880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775764693.806686:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764693.806690:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764693.806692:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764693.806698:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 426 00000100:00080000:0.0:1775764693.806703:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764693.806707:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764693.806720:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764693.806734:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861259008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764693.806816:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764693.806839:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861259136:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764693.807977:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861258112:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764693.808318:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861258880:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775764693.808392:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861259008:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775764693.813795:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861257728:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764693.813830:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861258496:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764693.815221:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764693.815236:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764693.815241:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861259008:192.168.202.104@tcp:38:kworker.0 00010000:00080000:1.0:1775764693.818933:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d09dca80 x1862023861259136/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764709 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764693.818958:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861259136:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764694.444312:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764695.471557:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764696.491865:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764697.516254:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764698.539869:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764698.987980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764698.987992:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764698.987995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764698.987999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764698.988004:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764698.988016:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 427 00000100:00080000:0.0:1775764698.988028:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764698.988035:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764698.988065:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764698.988100:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861259264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764698.988642:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861259264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775764698.988735:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861259392:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764698.999148:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764698.999164:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764698.999170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861259264:192.168.202.104@tcp:38:kworker.0 00010000:00080000:1.0:1775764699.004199:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09dd180 x1862023861259392/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764714 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764699.004220:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861259392:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764699.563929:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764700.590885:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764701.613719:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764702.638153:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764703.083930:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764693/real 1775764693] req@ffffa0e9d09df100 x1862023861258880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764703 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764703.083966:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861258880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764703 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764703.083991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764703.084025:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:3.0:1775764703.084028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861258880:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775764703.084120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764703.084126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775764703.084130:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764703.084138:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 427 00000100:00080000:2.0:1775764703.084153:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1775764703.084159:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775764703.084195:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861259520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764703.084224:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764703.084228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764703.084229:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764703.084252:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 432 00000100:00080000:2.0:1775764703.084256:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764703.084260:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764703.084272:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764703.084285:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861259648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764703.084339:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861259776:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764703.085919:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861259520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775764703.086085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861259648:192.168.202.104@tcp:38:kworker.0 00010000:00080000:2.0:1775764703.087833:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711500 x1862023861259776/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764719 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764703.087849:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861259776:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764703.091821:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764703.091838:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-11) 00000100:00100000:3.0:1775764703.091844:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861259648:192.168.202.104@tcp:38:kworker.0 00000100:00080000:3.0:1775764703.096260:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) MGC192.168.202.104@tcp: connect to target with instance 0 10000000:01000000:3.0:1775764703.096277:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1775764703.096287:0:2400:0:(import.c:1282:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.202.104@tcp) after server handle changed from 0x39b1144a2f76eaaa to 0x39b1144a2f76ed7b 00000100:00080000:3.0:1775764703.115405:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1775764703.115435:0:2400:0:(import.c:1667:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.202.104@tcp; invalidating 00000100:00100000:2.0:1775764703.117705:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861259520:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775764703.117883:0:11024:0:(import.c:1591:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.202.104@tcp to MGS@192.168.202.104@tcp 00000100:00100000:3.0:1775764703.117888:0:11024:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1775764703.117912:0:11024:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1775764703.117934:0:11024:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x39b1144a2f76eb1a expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764703.117947:0:11024:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x39b1144a2f76eb1a expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764703.117953:0:11024:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764703.117981:0:11024:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764703.117994:0:11024:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031400/0x64300da17553cbe7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x39b1144a2f76eb1a expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764703.118014:0:11024:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x39b1144a2f76eb83 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764703.118020:0:11024:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x39b1144a2f76eb83 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764703.118025:0:11024:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764703.118029:0:11024:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764703.118032:0:11024:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cbee lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x39b1144a2f76eb83 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764703.118046:0:11024:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0ea02053400/0x64300da17553cba8 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x39b1144a2f76eabf expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764703.118051:0:11024:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0ea02053400/0x64300da17553cba8 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x39b1144a2f76eabf expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764703.118056:0:11024:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764703.118059:0:11024:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764703.118062:0:11024:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0ea02053400/0x64300da17553cba8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x39b1144a2f76eabf expref: -99 pid: 6160 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764703.118084:0:11024:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x39b1144a2f76ec0f expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764703.118090:0:11024:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x39b1144a2f76ec0f expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764703.118095:0:11024:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1775764703.118098:0:11024:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764703.118102:0:11024:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030800/0x64300da17553cbfc lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x39b1144a2f76ec0f expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:3.0:1775764703.118115:0:11024:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:3.0:1775764703.118120:0:11024:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from RECOVER to FULL 10000000:01000000:3.0:1775764703.118122:0:11024:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1775764703.118125:0:11024:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:3.0:1775764703.118129:0:11024:0:(import.c:1737:ptlrpc_import_recovery_state_machine()) MGC192.168.202.104@tcp: Connection restored to 192.168.202.104@tcp (at 192.168.202.104@tcp) 00010000:00010000:1.0:1775764703.659875:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1775764704.242368:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1775764704.242376:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-cliir-ffffa0e9c80cb000 from 7 10000000:01000000:3.0:1775764704.242382:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1775764704.242463:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.242477:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.242510:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764704.242546:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861259904:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1775764704.242673:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d8600 going to sleep for 11 seconds 00000100:00100000:2.0:1775764704.256208:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861259904:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764704.256263:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764704.256279:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764704.256374:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260032:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1775764704.256501:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549f240 going to sleep for 11 seconds 00000100:00100000:3.0:1775764704.272304:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260032:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000020:00000080:3.0:1775764704.272360:0:6174:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 00000020:00000080:3.0:1775764704.272789:0:6174:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 00000020:00000080:3.0:1775764704.272840:0:6174:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 00000020:00000080:3.0:1775764704.272874:0:6174:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 10000000:01000000:3.0:1775764704.272956:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1775764704.272970:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.272983:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764704.272995:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1775764704.272998:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-client-ffffa0e9c80cb000 from 52 10000000:01000000:3.0:1775764704.273001:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1775764704.273045:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3800/0x64300da17553cc11 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.273052:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3800/0x64300da17553cc11 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.273064:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3800/0x64300da17553cc11 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764704.273088:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260160:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1775764704.273229:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d89c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764704.288553:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260160:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1775764704.288583:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3800/0x64300da17553cc11 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed89 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764704.288598:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3800/0x64300da17553cc11 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76ed89 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764704.288702:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260288:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1775764704.288767:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81f6180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764704.305054:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260288:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1775764704.305158:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260416:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000100:00100000:1.0:1775764704.305277:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81f6180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764704.328350:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260416:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1775764704.359366:0:11046:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 51 00000020:01000000:3.0:1775764704.365776:0:6174:0:(obd_config.c:2143:class_config_parse_llog()) Processed log lustre-client gen 52-51 (rc=0) 10000000:01000000:3.0:1775764704.365821:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1775764704.365845:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3800/0x64300da17553cc11 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed89 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.365860:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3800/0x64300da17553cc11 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed89 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764704.365873:0:6174:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1775764704.365876:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log params-ffffa0e9c80cb000 from 7 10000000:01000000:3.0:1775764704.365879:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1775764704.365931:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a2c00/0x64300da17553cc18 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.365939:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a2c00/0x64300da17553cc18 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764704.365954:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a2c00/0x64300da17553cc18 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764704.365994:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260544:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1775764704.366129:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c4f08cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764704.390198:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260544:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1775764704.390231:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a2c00/0x64300da17553cc18 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed90 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764704.390245:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a2c00/0x64300da17553cc18 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76ed90 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764704.390334:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260672:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1775764704.395592:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81f6180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764704.405811:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260672:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1775764704.405880:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260800:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000100:00100000:1.0:1775764704.405962:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81f6180 going to sleep for 11 seconds 00000100:00100000:0.0:1775764704.412609:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260800:192.168.202.104@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1775764704.413336:0:11054:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:0.0:1775764704.413394:0:6174:0:(obd_config.c:2143:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:0.0:1775764704.414807:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1775764704.414825:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a2c00/0x64300da17553cc18 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed90 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764704.414837:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a2c00/0x64300da17553cc18 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76ed90 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1775764704.414849:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1775764704.414851:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-sptlrpc-ffffa0e9d0958000 from 1 10000000:01000000:0.0:1775764704.414855:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1775764704.414897:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031a00/0x64300da17553cc1f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764704.414905:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031a00/0x64300da17553cc1f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764704.414917:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031a00/0x64300da17553cc1f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764704.414949:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260928:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764704.415051:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81e50c0 going to sleep for 11 seconds 00000100:00080000:3.0:1775764704.420550:0:11048:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffffa0e9d0199000 already not connected (conn 8, was 0): DISCONN 00010000:00080000:3.0:1775764704.420594:0:11048:0:(ldlm_lib.c:85:import_set_conn()) imp ffffa0e9d0199000@lustre-MDT0000-mdc-ffffa0e9c80cb000: found existing conn 192.168.202.104@tcp, moved to head 00000100:00080000:3.0:1775764704.420603:0:11048:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764704.420620:0:11048:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764704.420637:0:11048:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764704.420679:0:11048:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861261056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764704.422276:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861261056:192.168.202.104@tcp:38:lctl.0 00000100:00100000:0.0:1775764704.422695:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861260928:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1775764704.422722:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031a00/0x64300da17553cc1f lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76edeb expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764704.422735:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031a00/0x64300da17553cc1f lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76edeb expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764704.422809:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861261184:192.168.202.104@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1775764704.422849:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81e50c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764704.442064:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861261184:192.168.202.104@tcp:501:ll_cfg_requeue.0 10000000:01000000:0.0:1775764704.442135:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:0.0:1775764704.442146:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031a00/0x64300da17553cc1f lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76edeb expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764704.442156:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c0031a00/0x64300da17553cc1f lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76edeb expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00080000:1.0:1775764704.472472:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540e00 x1862023861261056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/416 e 0 to 0 dl 1775764729 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lctl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764704.472530:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa0e9c80cb000: committing for last_committed 4294967299 gen 1 00000100:00080000:1.0:1775764704.472556:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to target with instance 2 00000100:00080000:1.0:1775764704.472577:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:1.0:1775764704.472582:0:2400:0:(import.c:1694:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1775764704.472584:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa0e9c80cb000: committing for last_committed 4294967299 gen 1 00000100:00080000:1.0:1775764704.472587:0:2400:0:(recover.c:53:ptlrpc_replay_next()) import ffffa0e9d0199000 from lustre-MDT0000_UUID committed 4294967299 last 0 00000100:00080000:1.0:1775764704.472590:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:1.0:1775764704.472720:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861261056:192.168.202.104@tcp:38:lctl.0 00010000:00080000:1.0:1775764704.472743:0:11058:0:(ldlm_request.c:2960:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffffa0e9c80cb000 to lustre-MDT0000_UUID@192.168.202.104@tcp 00010000:00010000:1.0:1775764704.472747:0:11058:0:(ldlm_request.c:2876:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffffa0e9c80cb000 (0) 00010000:00010000:1.0:1775764704.472752:0:11058:0:(ldlm_request.c:2889:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffffa0e9c80cb000 00000100:00080000:1.0:1775764704.473164:0:11058:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:1.0:1775764704.473208:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861261312:192.168.202.104@tcp:400:ldlm_lock_repla.0 00010000:00010000:0.0:1775764704.684661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764705.713111:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764706.731846:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764707.756682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:2.0:1775764708.165039:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0d500 x1862023861261312/t0(0) o400->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764720 ref 1 fl Rpc:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764708.165064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:2.0:1775764708.165071:0:2400:0:(client.c:3204:ptlrpc_resend_req()) @@@ going to resend req@ffffa0e9c8d0d500 x1862023861261312/t0(0) o400->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764720 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764708.165081:0:2400:0:(client.c:3213:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa0e9c8d0d500 x1862023861261312/t0(0) o400->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764720 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764708.165089:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:1.0:1775764708.165722:0:2400:0:(import.c:1737:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffffa0e9c80cb000: Connection restored to 192.168.202.104@tcp (at 192.168.202.104@tcp) 00000100:00100000:1.0:1775764708.174028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861261312:192.168.202.104@tcp:400:ldlm_lock_repla.0 00000100:00100000:1.0:1775764708.205460:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861261568:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764708.206242:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861261440:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764708.206654:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861261696:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764708.220750:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861261568/t0(0) o400->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764724 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764708.220786:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861261568:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764708.221404:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861261440:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764708.238463:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0f480 x1862023861261696/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764724 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764708.238553:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861261696:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764708.780214:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764709.804107:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764710.827696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764711.852791:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764712.876060:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1775764713.068048:0:11135:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764713.068066:0:11135:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764713.068096:0:11135:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764713.068164:0:11135:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 11135 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764713.068175:0:11135:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 11135 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764713.068186:0:11135:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 11135 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764713.068215:0:11135:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861261824:192.168.202.104@tcp:101:lfs.0 00000100:00100000:0.0:1775764713.068610:0:11135:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81e5780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764713.128248:0:11135:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861261824:192.168.202.104@tcp:101:lfs.0 00010000:00010000:2.0:1775764713.128276:0:11135:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128290:0:11135:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128304:0:11135:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128316:0:11135:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128337:0:11135:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128346:0:11135:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128364:0:11135:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764713.128376:0:11135:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffffa0e9d09dfb80 x1862023861261824/t0(0) o101->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764729 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764713.128391:0:11135:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128405:0:11135:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa0e9c9b92c00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764713.128413:0:11135:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764713.128442:0:11135:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:2.0:1775764713.128449:0:11135:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764713.128454:0:11135:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764713.128457:0:11135:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b601f7fcb8 00010000:00010000:2.0:1775764713.128460:0:11135:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764713.128469:0:11135:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1775764713.134109:0:11135:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1775764713.134120:0:11135:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764713.134875:0:11135:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764713.134898:0:11135:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764713.134916:0:11135:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764713.134923:0:11135:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764713.134932:0:11135:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764713.134936:0:11135:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:3.0:1775764713.134943:0:11135:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b601f7fcb8 00010000:00010000:3.0:1775764713.134946:0:11135:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764713.134955:0:11135:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764713.135120:0:11135:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764713.135130:0:11135:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764713.135234:0:11135:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764713.135244:0:11135:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764713.135250:0:11135:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764713.135254:0:11135:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764713.135363:0:11135:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:3.0:1775764713.135370:0:11135:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b601f7fbb0 00010000:00010000:3.0:1775764713.135373:0:11135:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764713.135381:0:11135:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1775764713.135390:0:11135:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:3.0:1775764713.135398:0:11135:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:3.0:1775764713.137578:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.137608:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00800000:00000004:3.0:1775764713.137619:0:11135:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 285, max_age 445 00000100:00100000:3.0:1775764713.137672:0:11135:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861261952:192.168.202.104@tcp:41:lfs.0 00000100:00100000:3.0:1775764713.137785:0:11135:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c84c3540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764713.143246:0:11135:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861261952:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764713.143288:0:11135:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:3.0:1775764713.143376:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.143394:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00800000:00000004:3.0:1775764713.143402:0:11135:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 285, max_age 445 00000100:00100000:3.0:1775764713.143440:0:11135:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861262080:192.168.202.104@tcp:41:lfs.0 00000100:00100000:3.0:1775764713.143535:0:11135:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c84c3540 going to sleep for 11 seconds 00010000:00080000:3.0:1775764713.145674:0:11135:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0543b80 x1862023861262080/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764729 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764713.145691:0:11135:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861262080:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764713.145725:0:11135:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317443/353529 objects 1023743/1024000 00000080:00200080:3.0:1775764713.145796:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145811:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145819:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145822:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145827:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145830:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145834:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145838:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145841:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145844:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145848:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145851:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145855:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145858:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145862:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145866:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145869:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145873:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145877:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145880:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145883:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145887:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145890:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145893:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145905:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145910:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145915:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145920:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145924:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145927:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145930:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145933:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145937:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145940:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145944:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145947:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145950:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145954:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145958:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145961:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145965:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145968:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145972:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145975:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145979:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145982:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145987:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.145990:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.145994:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146004:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146012:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146017:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146020:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146025:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146030:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146033:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146037:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146040:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146043:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146046:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146049:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146052:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146056:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146059:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146063:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146066:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146070:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146074:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146078:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146081:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146085:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146088:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146092:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146094:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146106:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146111:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146115:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146119:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146123:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146126:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146129:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146133:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146137:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146140:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146144:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146147:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146151:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146154:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146158:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146160:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146164:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146167:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146170:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146173:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146177:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146180:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146184:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146187:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146191:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146201:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146206:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146210:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146214:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146217:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146221:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146224:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146228:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146231:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146234:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146237:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146241:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146244:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146248:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146251:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146255:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146258:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146262:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146280:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146285:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146289:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146293:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146296:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146299:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146303:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146314:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00800000:00000080:3.0:1775764713.146318:0:11135:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00000080:00200080:3.0:1775764713.146323:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:3.0:1775764713.146336:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00020000:00000004:3.0:1775764713.146345:0:11135:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 285, max_age 445 00000100:00080000:3.0:1775764713.146359:0:11135:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 282 00000100:00080000:3.0:1775764713.146370:0:11135:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764713.146381:0:11135:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8437760, lost: 0. 00000100:00100000:3.0:1775764713.146403:0:11135:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861262208/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764713.146444:0:11135:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa0e9d0543b80 x1862023861262336/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764713.146453:0:11135:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c84c3540 going to sleep for 0 seconds 00000100:00100000:1.0:1775764713.147161:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861262208:192.168.202.104@tcp:8:lfs.0 00010000:00080000:2.0:1775764713.149257:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542d80 x1862023861262208/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764723 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764713.149282:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffffa0e9c80cb000: committing for last_committed 0 gen 2 00000100:00080000:2.0:1775764713.149292:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to target with instance 1 00000100:00080000:2.0:1775764713.149301:0:2400:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffffa0e9c80cb000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:2.0:1775764713.149324:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1775764713.149337:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1775764713.149342:0:2400:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:2.0:1775764713.149346:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:2.0:1775764713.149353:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1775764713.149355:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1775764713.149362:0:2400:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000008b046916): req@ffffa0e9d0543b80 x1862023861262336/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764713.149386:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861262208:192.168.202.104@tcp:8:lfs.0 00000100:00100000:3.0:1775764713.149598:0:11135:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c84c3540 going to sleep for 11 seconds 00010000:00080000:3.0:1775764713.173084:0:11135:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0543b80 x1862023861262336/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764729 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764713.173104:0:11135:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861262336:192.168.202.104@tcp:13:lfs.0 00020000:00000004:3.0:1775764713.173127:0:11135:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901369/958279 objects 261744/262144 00000080:00200080:3.0:1775764713.173193:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:3.0:1775764713.173210:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c6000 uarg=0000000000000000 00020000:00000004:3.0:1775764713.173225:0:11135:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 286, max_age 445 00000100:00080000:3.0:1775764713.173236:0:11135:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 282 00000100:00080000:3.0:1775764713.173244:0:11135:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764713.173254:0:11135:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8437760, lost: 0. 00000100:00100000:3.0:1775764713.173295:0:11135:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861262464/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764713.173359:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861262464:192.168.202.104@tcp:8:lfs.0 00000100:00080000:3.0:1775764713.175562:0:11135:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa0e9d0543b80 x1862023861262592/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764713.175575:0:11135:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c84c3540 going to sleep for 0 seconds 00010000:00080000:0.0:1775764713.184553:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0543800 x1862023861262464/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764723 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764713.184568:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffffa0e9c80cb000: committing for last_committed 0 gen 2 00000100:00080000:0.0:1775764713.184577:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to target with instance 1 00000100:00080000:0.0:1775764713.184586:0:2400:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffffa0e9c80cb000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:0.0:1775764713.184597:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:0.0:1775764713.184602:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:0.0:1775764713.184608:0:2400:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:0.0:1775764713.184611:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:0.0:1775764713.184616:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:0.0:1775764713.184618:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:0.0:1775764713.184625:0:2400:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000008b046916): req@ffffa0e9d0543b80 x1862023861262592/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764713.190699:0:11135:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c84c3540 going to sleep for 11 seconds 00000100:00100000:0.0:1775764713.193140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861262464:192.168.202.104@tcp:8:lfs.0 00010000:00080000:2.0:1775764713.200367:0:11135:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0543b80 x1862023861262592/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764729 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764713.200387:0:11135:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11135:1862023861262592:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764713.200416:0:11135:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901369/958279 objects 261744/262144 00000080:00200080:2.0:1775764713.200527:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200546:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200554:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200558:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200561:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200565:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200569:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200573:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200577:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200580:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200583:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200604:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200609:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200613:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200616:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200620:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200623:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200626:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200629:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200632:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200636:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200639:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200643:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200646:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200650:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200653:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200656:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200659:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200662:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200665:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200668:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200671:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200674:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200677:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200680:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200683:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200694:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200698:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200701:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200704:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200707:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200711:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200714:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200717:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200721:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200724:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200727:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200731:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200734:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200738:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200741:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200744:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200749:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200752:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200769:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200773:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200778:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200781:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200786:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200789:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200793:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200803:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200808:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200812:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200815:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200819:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200823:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200826:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200830:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200834:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200837:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200841:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200844:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200847:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200850:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200853:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200856:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200859:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200862:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200866:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200870:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200873:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200877:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200881:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200884:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200887:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200898:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200902:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200905:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200920:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200925:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200928:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200931:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200934:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200937:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200939:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200943:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200946:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200949:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200951:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200955:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200957:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200960:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200963:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200967:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200970:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200973:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200976:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200979:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200982:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.200985:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.200995:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.201000:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.201003:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.201007:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.201010:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.201013:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.201017:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.201021:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.201024:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.201028:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.201031:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.201035:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.201038:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200080:2.0:1775764713.201041:0:11135:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffdce4954b0 00020000:00000080:2.0:1775764713.201045:0:11135:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133400 uarg=0000000000000000 00000080:00200008:2.0:1775764713.201061:0:11135:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000100:00100000:2.0:1775764713.327400:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861262720:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764713.340464:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861262720:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764713.899691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764714.923813:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764715.947841:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764716.971879:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764718.000085:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764718.002317:0:11388:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg204-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00000100:00100000:2.0:1775764718.443864:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861262848:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764718.444055:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861263104:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764718.444090:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861263360:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764718.444125:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764718.444132:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861262976:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764718.444168:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861263232:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764718.451639:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713800 x1862023861263104/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764734 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764718.451667:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861263104:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764718.451704:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861262976:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764718.452346:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711500 x1862023861263232/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764734 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764718.452358:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861263232:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764718.452371:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713480 x1862023861263360/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764734 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764718.452382:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861263360:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764718.457772:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861262848:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764719.020765:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764720.030144:0:11544:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:3.0:1775764720.047691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764721.070965:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764722.099632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1775764722.187444:0:11582:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764722.187465:0:11582:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764722.187528:0:11582:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764722.187558:0:11582:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:0.0:1775764722.187580:0:11582:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:0.0:1775764722.187590:0:11582:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764722.187600:0:11582:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764722.187605:0:11582:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764722.187614:0:11582:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d7fcb8 00010000:00010000:0.0:1775764722.187618:0:11582:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:0.0:1775764722.187628:0:11582:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200008:0.0:1775764722.189895:0:11582:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1775764722.189909:0:11582:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764722.189940:0:11582:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764722.189964:0:11582:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:0.0:1775764722.190000:0:11582:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:0.0:1775764722.190009:0:11582:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764722.190018:0:11582:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764722.190022:0:11582:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764722.190030:0:11582:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d7fcb8 00010000:00010000:0.0:1775764722.190035:0:11582:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:0.0:1775764722.190045:0:11582:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:0.0:1775764722.190236:0:11582:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764722.190258:0:11582:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764722.190266:0:11582:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:0.0:1775764722.190275:0:11582:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:0.0:1775764722.190282:0:11582:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764722.190287:0:11582:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764722.190290:0:11582:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764722.190294:0:11582:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d7fbb0 00010000:00010000:0.0:1775764722.190297:0:11582:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:0.0:1775764722.190311:0:11582:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:0.0:1775764722.190320:0:11582:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:0.0:1775764722.190330:0:11582:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:0.0:1775764722.190651:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.190671:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00800000:00000004:0.0:1775764722.190681:0:11582:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 446, max_age 454 00000100:00100000:0.0:1775764722.190738:0:11582:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263488:192.168.202.104@tcp:41:lfs.0 00000100:00100000:0.0:1775764722.190870:0:11582:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d0949900 going to sleep for 11 seconds 00000100:00100000:0.0:1775764722.202239:0:11582:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263488:192.168.202.104@tcp:41:lfs.0 00800000:00000004:0.0:1775764722.202296:0:11582:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:0.0:1775764722.202416:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.202444:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00800000:00000004:0.0:1775764722.202456:0:11582:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 446, max_age 454 00000100:00100000:0.0:1775764722.202554:0:11582:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263616:192.168.202.104@tcp:41:lfs.0 00000100:00100000:0.0:1775764722.202779:0:11582:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d0949900 going to sleep for 11 seconds 00010000:00080000:0.0:1775764722.209970:0:11582:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09de300 x1862023861263616/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764738 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764722.209997:0:11582:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263616:192.168.202.104@tcp:41:lfs.0 00800000:00000004:0.0:1775764722.210030:0:11582:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317443/353529 objects 1023743/1024000 00000080:00200080:0.0:1775764722.210126:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210151:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210180:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210185:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210189:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210193:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210197:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210200:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210203:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210206:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210210:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210212:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210216:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210219:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210222:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210226:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210229:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210232:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210235:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210238:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210241:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210253:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210260:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210264:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210267:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210270:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210274:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210283:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210289:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210292:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210297:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210300:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210305:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210308:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210311:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210314:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210317:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210321:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210324:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210327:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210330:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210333:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210337:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210341:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210344:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210348:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210352:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210355:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210360:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210363:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210367:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210370:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210382:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210386:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210390:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210393:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210397:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210400:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210405:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210408:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210412:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210415:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210420:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210424:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210428:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210431:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210435:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210439:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210443:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210446:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210450:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210453:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210458:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210461:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210465:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210469:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210472:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210481:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210517:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210523:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210528:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210532:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210536:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210540:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210543:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210547:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210551:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210555:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210559:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210562:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210566:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210569:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210573:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210576:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210579:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210582:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210585:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210588:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210592:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210595:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210598:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210602:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210613:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210617:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210621:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210624:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210628:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210631:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210636:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210639:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210643:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210647:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210651:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210654:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210658:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210661:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210665:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210668:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:0.0:1775764722.210672:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:0.0:1775764722.210678:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d76800 uarg=0000000000000000 00000080:00200080:1.0:1775764722.211717:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:1.0:1775764722.211739:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c2d8f800 uarg=0000000000000000 00000080:00200080:1.0:1775764722.211748:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:1.0:1775764722.211752:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c2d8f800 uarg=0000000000000000 00000080:00200080:1.0:1775764722.211756:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00800000:00000080:1.0:1775764722.211760:0:11582:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c2d8f800 uarg=0000000000000000 00000080:00200080:1.0:1775764722.211775:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:1.0:1775764722.211784:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c2d8f800 uarg=0000000000000000 00020000:00000004:1.0:1775764722.211795:0:11582:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 446, max_age 454 00000100:00100000:1.0:1775764722.211875:0:11582:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263744:192.168.202.104@tcp:13:lfs.0 00000100:00100000:1.0:1775764722.211985:0:11582:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81f6240 going to sleep for 11 seconds 00010000:00080000:3.0:1775764722.218664:0:11582:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861263744/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764738 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764722.218693:0:11582:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263744:192.168.202.104@tcp:13:lfs.0 00020000:00000004:3.0:1775764722.218728:0:11582:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901369/958279 objects 261744/262144 00000080:00200080:3.0:1775764722.218828:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.218862:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00020000:00000004:3.0:1775764722.218873:0:11582:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 446, max_age 454 00000100:00100000:3.0:1775764722.218932:0:11582:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263872:192.168.202.104@tcp:13:lfs.0 00000100:00100000:3.0:1775764722.219020:0:11582:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d63c0 going to sleep for 11 seconds 00010000:00080000:3.0:1775764722.231603:0:11582:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0541c00 x1862023861263872/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764738 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764722.231630:0:11582:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11582:1862023861263872:192.168.202.104@tcp:13:lfs.0 00020000:00000004:3.0:1775764722.231984:0:11582:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901369/958279 objects 261744/262144 00000080:00200080:3.0:1775764722.232419:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232446:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232456:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232461:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232465:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232468:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232472:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232476:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232479:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232483:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232529:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232533:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232537:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232540:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232543:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232547:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232550:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232554:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232557:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232560:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232564:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232568:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232572:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232576:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232579:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232591:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232596:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232599:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232602:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232605:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232609:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232612:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232615:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232618:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232621:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232624:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232627:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232630:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232633:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232636:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232640:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232643:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232647:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232651:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232654:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232658:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232661:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232664:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232667:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232670:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232681:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232685:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232689:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232693:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232697:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232701:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232705:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232708:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232712:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232715:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232720:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232723:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232726:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232730:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232735:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232738:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232742:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232745:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232749:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232753:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232757:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232760:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232764:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232768:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232772:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232782:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232788:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232791:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232795:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232798:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232802:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232805:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232809:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232812:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232815:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232818:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232822:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232826:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232830:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232834:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232838:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232848:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232853:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232857:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232861:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232864:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232868:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232871:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232875:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232879:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232891:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232896:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232899:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232903:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232906:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232910:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232913:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232916:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232920:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232923:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232927:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232929:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232932:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232936:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232940:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232943:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232946:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232949:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232953:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232955:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232959:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232962:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232966:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232969:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764722.232973:0:11582:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc3e5af050 00020000:00000080:3.0:1775764722.232982:0:11582:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200008:3.0:1775764722.233002:0:11582:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:3.0:1775764723.115793:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764723.564087:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861264000:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764723.564186:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264256:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764723.564210:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264512:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764723.564244:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764723.564249:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264128:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764723.564286:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264384:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764723.570862:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861264384/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764739 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764723.570883:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264384:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764723.575359:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861264000:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764723.575388:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0ea00 x1862023861264256/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764739 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764723.575401:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264256:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764723.575414:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0f480 x1862023861264512/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764739 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764723.575449:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264512:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764723.575603:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861264128:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764724.139824:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764725.164750:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764726.187744:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764727.212582:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764728.235756:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764728.684593:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861264640:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764728.684746:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861264896:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764728.684781:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265152:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764728.684826:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1775764728.684834:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861264768:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764728.684861:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265024:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764728.697169:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861264768:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764728.697209:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09dc000 x1862023861265024/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764744 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764728.697224:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265024:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764728.697236:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861264640:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764728.697248:0:2401:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-MDT0001-mdc-ffffa0e9c80cb000 of lustre-MDT0001_UUID@192.168.202.104@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1775764728.697254:0:2401:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1775764728.697259:0:2401:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffffa0e9c80cb000: Connection to lustre-MDT0001 (at 192.168.202.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1775764728.715266:0:2401:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764728.715297:0:2401:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 281 00000100:00080000:0.0:1775764728.715331:0:2401:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775764728.715676:0:2401:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764728.715742:0:2401:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8d0f800 x1862023861265280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764728.715779:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861264896:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764728.715805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861265280:192.168.202.104@tcp:38:ptlrpcd_00_00.0 00010000:00080000:1.0:1775764728.715827:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09ddc00 x1862023861265152/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764744 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764728.715839:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265152:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764728.732032:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764728.732053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:0.0:1775764728.732060:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861265280:192.168.202.104@tcp:38:ptlrpcd_00_00.0 00010000:00010000:3.0:1775764729.259708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764730.287211:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764731.307876:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764732.331881:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764733.357939:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764733.804255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764733.804265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775764733.804271:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764733.804286:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 462 00000100:00080000:1.0:1775764733.804304:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1775764733.804313:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775764733.804327:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764733.804358:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8d0e680 x1862023861265664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764733.804430:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265536:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764733.804704:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861265664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775764733.804852:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861265408:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764733.804936:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265920:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764733.806259:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861265792:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764733.812679:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861265408:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764733.816564:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9c8d0d500 x1862023861265920/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764749 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764733.817067:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265920:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764733.821609:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764733.821629:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764733.821636:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861265664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775764733.821818:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861265536:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764733.833980:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9c8d0df80 x1862023861265792/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764749 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764733.834018:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861265792:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764734.384858:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764735.409813:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764736.427881:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764737.452947:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764738.478375:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764738.923952:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861266048:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764738.924136:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764738.924144:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861266176:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764738.924405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764738.924411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775764738.924426:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764738.924433:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 467 00000100:00080000:0.0:1775764738.924441:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1775764738.924449:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764738.924460:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764738.924482:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861266304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764738.924586:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861266432:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764738.924711:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764738.924712:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861266304:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775764738.924722:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861266560:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764738.935913:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861266176:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764738.938371:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861266048:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764738.939225:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09de300 x1862023861266432/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764754 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764738.939240:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861266432:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764738.943954:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09dc000 x1862023861266560/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764754 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764738.943974:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861266560:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764738.952025:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764738.952044:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764738.952065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861266304:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764739.500654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764740.524880:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764741.547837:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764742.574170:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764743.601124:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764744.045772:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861266688:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764744.046918:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861266816:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764744.047067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764744.047073:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775764744.047077:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764744.047084:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 472 00000100:00080000:0.0:1775764744.047093:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1775764744.047101:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764744.047115:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764744.047140:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861266944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764744.047201:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861266944:192.168.202.104@tcp:38:kworker.0 00000100:00080000:0.0:1775764744.047265:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:0.0:1775764744.047269:0:8:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffffa0e9c80cb000: disconnect after 22s idle 00000100:00100000:1.0:1775764744.059118:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861266688:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764744.066073:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:0.0:1775764744.066081:0:8:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffffa0e9c80cb000: disconnect after 22s idle 00000100:00100000:0.0:1775764744.066198:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861266816:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764744.069175:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764744.069194:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764744.069200:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861266944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764744.069247:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267072:192.168.202.104@tcp:9:kworker.0 00000100:00100000:2.0:1775764744.069581:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267200:192.168.202.104@tcp:9:kworker.0 00010000:00080000:2.0:1775764744.078057:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09dfb80 x1862023861267072/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764754 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764744.078084:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9d09dfb80 x1862023861267072/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764754 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764744.078093:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1775764744.078098:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267072:192.168.202.104@tcp:9:kworker.0 00010000:00080000:2.0:1775764744.078114:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09df100 x1862023861267200/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764754 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764744.078124:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9d09df100 x1862023861267200/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764754 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764744.078130:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1775764744.078152:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267200:192.168.202.104@tcp:9:kworker.0 00010000:00010000:0.0:1775764744.623763:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764745.644239:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764746.078314:0:2393:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.104@tcp (source: 12345-192.168.202.104@tcp) 00000100:00100000:0.0:1775764746.078440:0:6161:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862023885527680 00000100:00100000:0.0:1775764746.078468:0:6161:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862023885527680 00000100:00100000:0.0:1775764746.078559:0:6161:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.104@tcp, seq: 4 00000100:00100000:0.0:1775764746.078568:0:6161:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffffa0e9c8d0f480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:14120:x1862023885527680:12345-192.168.202.104@tcp:104: 00010000:00010000:0.0:1775764746.078590:0:6161:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764746.078684:0:6161:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764746.078689:0:6161:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffffa0e9c8d0f480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:14120:x1862023885527680:12345-192.168.202.104@tcp:104: Request processed in 122us (393us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764746.078695:0:6161:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.104@tcp, seq: 4 00010000:00010000:2.0:1775764746.078763:0:6650:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764746.078771:0:6650:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764746.078777:0:6650:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0e9c84a3e00 already unused, calling callback (ffffffffc112fc20) 10000000:00010000:2.0:1775764746.078781:0:6650:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764746.078789:0:6650:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1775764746.078794:0:6650:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764746.078798:0:6650:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1775764746.078842:0:6650:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764746.078847:0:6650:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764746.078872:0:6650:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764746.078878:0:6650:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c84a3e00/0x64300da17553cc0a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76ed82 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764746.078897:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764746.078898:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764746.078899:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764746.078908:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764746.078929:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861267328:192.168.202.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764746.088458:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861267328:192.168.202.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764746.667873:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764747.706832:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1775764747.947610:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:0.0:1775764747.947622:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-cliir-ffffa0e9c80cb000 from 11 10000000:01000000:0.0:1775764747.947630:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:0.0:1775764747.947700:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764747.947715:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764747.947730:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764747.947766:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861267456:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764747.947895:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d0949480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764747.958922:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861267456:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764747.958977:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764747.958997:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764747.959124:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861267584:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1775764747.959262:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549f600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764747.992407:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861267584:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000020:00000080:1.0:1775764747.992509:0:6174:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 10000000:01000000:1.0:1775764747.992719:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:1.0:1775764747.992735:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764747.992752:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:0.0:1775764748.045480:0:11832:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffffa0e9d04cb000 already not connected (conn 5, was 0): DISCONN 00010000:00080000:0.0:1775764748.046681:0:11832:0:(ldlm_lib.c:85:import_set_conn()) imp ffffa0e9d04cb000@lustre-MDT0001-mdc-ffffa0e9c80cb000: found existing conn 192.168.202.104@tcp, moved to head 00000100:00080000:0.0:1775764748.046690:0:11832:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764748.046702:0:11832:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 0 00000100:00080000:0.0:1775764748.046735:0:11832:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764748.046753:0:11832:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764748.046796:0:11832:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861267712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764748.046884:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267712:192.168.202.104@tcp:38:lctl.0 00010000:00080000:3.0:1775764748.072406:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dd180 x1862023861267712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/416 e 0 to 0 dl 1775764758 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lctl.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764748.072435:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to target with instance 2 00000100:00080000:3.0:1775764748.072458:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1775764748.072464:0:2400:0:(import.c:1694:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0001_UUID 00000100:00080000:3.0:1775764748.072467:0:2400:0:(recover.c:53:ptlrpc_replay_next()) import ffffa0e9d04cb000 from lustre-MDT0001_UUID committed 0 last 0 00000100:00080000:3.0:1775764748.072471:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:3.0:1775764748.076221:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267712:192.168.202.104@tcp:38:lctl.0 00010000:00080000:2.0:1775764748.077611:0:11837:0:(ldlm_request.c:2960:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0001-mdc-ffffa0e9c80cb000 to lustre-MDT0001_UUID@192.168.202.104@tcp 00010000:00010000:2.0:1775764748.077619:0:11837:0:(ldlm_request.c:2876:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0001-mdc-ffffa0e9c80cb000 (0) 00010000:00010000:2.0:1775764748.077627:0:11837:0:(ldlm_request.c:2889:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0001-mdc-ffffa0e9c80cb000 00000100:00080000:2.0:1775764748.078089:0:11837:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:3.0:1775764748.078919:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267840:192.168.202.104@tcp:400:ldlm_lock_repla.0 00010000:00010000:2.0:1775764748.715993:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764749.164673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level REPLAY_WAIT/7 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764749.164680:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: REPLAY_WAIT) 00000100:00100000:3.0:1775764749.164786:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861267968:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764749.164956:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764749.164963:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861268096:192.168.202.104@tcp:400:kworker.0 00010000:00010000:3.0:1775764749.172443:0:2404:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1775764749.172476:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861268096:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764749.172548:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861267968:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764749.175317:0:6172:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764749.175352:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764749.175353:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764749.175355:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764749.745367:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764750.763765:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:3.0:1775764751.128311:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713480 x1862023861267840/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764764 ref 1 fl Rpc:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764751.128337:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:3.0:1775764751.128344:0:2400:0:(client.c:3204:ptlrpc_resend_req()) @@@ going to resend req@ffffa0e9d0713480 x1862023861267840/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764764 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764751.128356:0:2400:0:(client.c:3213:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa0e9d0713480 x1862023861267840/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764764 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764751.128362:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from RECOVER to FULL 00000100:02000000:0.0:1775764751.129293:0:2400:0:(import.c:1737:ptlrpc_import_recovery_state_machine()) lustre-MDT0001-mdc-ffffa0e9c80cb000: Connection restored to 192.168.202.104@tcp (at 192.168.202.104@tcp) 00000100:00100000:0.0:1775764751.157069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861267840:192.168.202.104@tcp:400:ldlm_lock_repla.0 00010000:00010000:1.0:1775764751.788457:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764752.813025:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764753.195837:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861268224:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764753.198114:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861268352:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764753.198284:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861268480:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764753.218577:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861268224:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764753.225892:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861268352:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764753.225972:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542300 x1862023861268480/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764769 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764753.225986:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861268480:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764753.838309:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764754.866089:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764755.883767:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764756.926675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764757.308965:0:11896:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764757.308978:0:11896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764757.308999:0:11896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764757.309017:0:11896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764757.309032:0:11896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:2.0:1775764757.309040:0:11896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764757.309047:0:11896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764757.309050:0:11896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764757.309057:0:11896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b601067cb8 00010000:00010000:2.0:1775764757.309060:0:11896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764757.309069:0:11896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200008:2.0:1775764757.312047:0:11896:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764757.312058:0:11896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764757.312081:0:11896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764757.312099:0:11896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764757.312115:0:11896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:2.0:1775764757.312131:0:11896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764757.312138:0:11896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764757.312142:0:11896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764757.312149:0:11896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b601067cb8 00010000:00010000:2.0:1775764757.312152:0:11896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764757.312161:0:11896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1775764757.312288:0:11896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764757.312296:0:11896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764757.315650:0:11896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764757.315672:0:11896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1775764757.315683:0:11896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764757.315692:0:11896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764757.315697:0:11896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764757.315705:0:11896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b601067bb0 00010000:00010000:2.0:1775764757.315709:0:11896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764757.315718:0:11896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1775764757.315749:0:11896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764757.315761:0:11896:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:2.0:1775764757.315930:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.315948:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00800000:00000004:2.0:1775764757.315956:0:11896:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 455, max_age 489 00000100:00100000:2.0:1775764757.316008:0:11896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11896:1862023861268608:192.168.202.104@tcp:41:lfs.0 00000100:00100000:2.0:1775764757.316114:0:11896:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549fc00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764757.324345:0:11896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11896:1862023861268608:192.168.202.104@tcp:41:lfs.0 00800000:00000004:2.0:1775764757.324371:0:11896:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:2.0:1775764757.324452:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.324468:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00800000:00000004:2.0:1775764757.324475:0:11896:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 455, max_age 489 00000100:00100000:2.0:1775764757.324540:0:11896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11896:1862023861268736:192.168.202.104@tcp:41:lfs.0 00000100:00100000:2.0:1775764757.324605:0:11896:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549fc00 going to sleep for 11 seconds 00010000:00080000:2.0:1775764757.327141:0:11896:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713800 x1862023861268736/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764773 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764757.327161:0:11896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11896:1862023861268736:192.168.202.104@tcp:41:lfs.0 00800000:00000004:2.0:1775764757.327182:0:11896:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317437/353529 objects 1023743/1024000 00000080:00200080:2.0:1775764757.327233:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327247:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327254:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327258:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327278:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327283:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327287:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327290:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327294:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327297:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327313:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327317:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327321:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327324:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327328:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327331:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327335:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327339:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327342:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327345:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327349:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327352:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327355:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327357:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327361:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327364:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327367:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327370:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327374:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327385:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327390:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327393:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327397:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327400:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327404:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327407:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327410:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327414:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327419:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327422:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327426:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327429:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327433:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327436:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327439:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327443:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327447:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327450:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327454:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327457:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327460:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327463:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327467:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327470:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327481:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327502:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327508:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327512:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327515:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327519:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327523:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327526:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327530:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327533:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327537:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327541:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327545:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327548:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327551:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327554:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327558:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327562:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327565:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327568:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327571:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327574:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327578:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327581:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327584:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327594:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327599:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327602:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327605:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327608:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327612:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327615:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327618:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327621:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327624:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327628:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327632:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327634:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327638:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327641:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327644:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327647:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327650:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327653:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327656:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327659:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327662:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327666:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327669:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327672:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327683:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327687:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327691:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327694:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327698:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327701:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327704:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327707:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327711:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327714:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327718:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327721:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327725:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327728:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327731:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327735:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327738:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327741:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327745:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327748:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327751:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00800000:00000080:2.0:1775764757.327754:0:11896:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.327757:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.327763:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00020000:00000004:2.0:1775764757.327770:0:11896:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 455, max_age 489 00000100:00080000:2.0:1775764757.327788:0:11896:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 446 00000100:00080000:2.0:1775764757.327798:0:11896:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764757.327807:0:11896:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8437760, lost: 0. 00000100:00100000:2.0:1775764757.327828:0:11896:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861268864/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764757.327869:0:11896:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa0e9d0713800 x1862023861268992/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764757.327878:0:11896:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549fc00 going to sleep for 0 seconds 00000100:00100000:0.0:1775764757.328032:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861268864:192.168.202.104@tcp:8:lfs.0 00010000:00080000:3.0:1775764757.337653:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0712680 x1862023861268864/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764767 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764757.337670:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffffa0e9c80cb000: committing for last_committed 0 gen 3 00000100:00080000:3.0:1775764757.337679:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to target with instance 1 00000100:00080000:3.0:1775764757.337688:0:2400:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffffa0e9c80cb000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:3.0:1775764757.337697:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764757.337702:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764757.337707:0:2400:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:3.0:1775764757.337710:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:3.0:1775764757.337728:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764757.337730:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764757.337738:0:2400:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000009d1fcde6): req@ffffa0e9d0713800 x1862023861268992/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764757.337765:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861268864:192.168.202.104@tcp:8:lfs.0 00000100:00100000:2.0:1775764757.338234:0:11896:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549fc00 going to sleep for 11 seconds 00010000:00080000:2.0:1775764757.345871:0:11896:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713800 x1862023861268992/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764773 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764757.345888:0:11896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11896:1862023861268992:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764757.345909:0:11896:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901369/958279 objects 261712/262144 00000080:00200080:2.0:1775764757.346258:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.346275:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00020000:00000004:2.0:1775764757.346282:0:11896:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 455, max_age 489 00000100:00080000:2.0:1775764757.346294:0:11896:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 446 00000100:00080000:2.0:1775764757.346304:0:11896:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764757.346314:0:11896:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8437760, lost: 0. 00000100:00100000:2.0:1775764757.346334:0:11896:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861269120/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764757.346367:0:11896:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa0e9d0713800 x1862023861269248/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764757.346375:0:11896:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549fc00 going to sleep for 0 seconds 00000100:00100000:3.0:1775764757.346453:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861269120:192.168.202.104@tcp:8:lfs.0 00010000:00080000:3.0:1775764757.353309:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713100 x1862023861269120/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764767 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764757.353324:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffffa0e9c80cb000: committing for last_committed 0 gen 3 00000100:00080000:3.0:1775764757.353334:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to target with instance 1 00000100:00080000:3.0:1775764757.353343:0:2400:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffffa0e9c80cb000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:3.0:1775764757.353353:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764757.353359:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764757.353365:0:2400:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:3.0:1775764757.353368:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:3.0:1775764757.353374:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764757.353376:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764757.353383:0:2400:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000009d1fcde6): req@ffffa0e9d0713800 x1862023861269248/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764757.353410:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861269120:192.168.202.104@tcp:8:lfs.0 00000100:00100000:2.0:1775764757.353583:0:11896:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c549fc00 going to sleep for 11 seconds 00010000:00080000:2.0:1775764757.364197:0:11896:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713800 x1862023861269248/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764773 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764757.364218:0:11896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:11896:1862023861269248:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764757.364243:0:11896:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901369/958279 objects 261712/262144 00000080:00200080:2.0:1775764757.364319:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364337:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364344:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364349:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364353:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364357:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364361:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364364:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364368:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364371:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364375:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364379:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364383:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364387:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364390:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364393:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364397:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364400:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364404:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364408:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364428:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364432:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364435:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364438:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364441:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364444:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364447:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364450:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364453:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364456:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364460:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364464:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364468:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364472:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364476:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364479:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364482:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364529:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364535:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364539:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364543:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364546:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364550:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364554:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364557:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364568:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364573:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364577:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364581:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364584:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364587:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364590:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364593:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364596:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364599:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364602:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364606:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364609:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364613:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364616:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364620:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364623:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364628:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364631:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364634:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364637:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364641:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364644:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364647:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364651:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364663:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364667:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364670:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364673:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364677:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364680:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364684:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364687:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364691:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364693:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364697:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364700:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364704:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364707:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364710:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364714:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364717:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364721:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364724:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364726:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364731:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364734:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364737:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364740:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364744:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364753:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364758:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364761:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364764:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364767:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364770:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364773:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364776:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364779:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364782:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364785:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364789:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364792:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364795:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364798:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364802:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364805:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364809:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364812:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364816:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364819:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364823:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364827:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364831:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364834:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364846:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364850:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364853:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364857:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764757.364860:0:11896:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe23448de0 00020000:00000080:2.0:1775764757.364864:0:11896:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200008:2.0:1775764757.364878:0:11896:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:1.0:1775764757.931643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764758.381356:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861269504:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764758.381370:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861269376:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764758.381442:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861269632:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764758.384479:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861269760:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764758.384564:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861269888:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764758.389249:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861269376:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764758.394158:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861269504:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764758.394179:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0d500 x1862023861269760/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764774 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764758.394191:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861269760:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764758.394293:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0e680 x1862023861269632/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764774 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764758.394319:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861269632:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764758.394332:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0f800 x1862023861269888/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764774 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764758.394341:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861269888:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764758.959269:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764759.984130:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764761.004802:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764762.028822:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764762.959549:0:12148:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg204-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0001-mdc-*.mds_server_uuid 00010000:00010000:2.0:1775764763.052831:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764763.500670:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270144:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764763.500766:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270400:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764763.500803:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1775764763.500809:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861270016:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764763.500838:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270272:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764763.500860:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270528:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764763.509756:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861270272/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764763.509774:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270272:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764763.509787:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0ea00 x1862023861270528/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764763.509795:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270528:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764763.509806:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270144:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764763.509816:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0d500 x1862023861270400/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764763.509824:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861270400:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764763.513336:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861270016:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764764.099251:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764765.101468:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1775764765.877228:0:12305:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0001-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:2.0:1775764766.126256:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764767.147666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764768.153943:0:12343:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764768.153959:0:12343:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764768.153988:0:12343:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764768.154008:0:12343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1775764768.154024:0:12343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1775764768.154032:0:12343:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764768.154039:0:12343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764768.154042:0:12343:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764768.154086:0:12343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6018afcb8 00010000:00010000:2.0:1775764768.154091:0:12343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1775764768.154113:0:12343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200008:2.0:1775764768.154985:0:12343:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764768.154991:0:12343:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764768.155001:0:12343:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764768.155011:0:12343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764768.155023:0:12343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1775764768.155028:0:12343:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764768.155032:0:12343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764768.155035:0:12343:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764768.155040:0:12343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6018afcb8 00010000:00010000:2.0:1775764768.155042:0:12343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764768.155049:0:12343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764768.159869:0:12343:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764768.159904:0:12343:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764768.159924:0:12343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764768.159961:0:12343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764768.159970:0:12343:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764768.159978:0:12343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764768.159982:0:12343:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764768.159990:0:12343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6018afbb0 00010000:00010000:2.0:1775764768.159994:0:12343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764768.160003:0:12343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764768.160019:0:12343:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764768.160030:0:12343:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:2.0:1775764768.160685:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.160705:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8133c00 uarg=0000000000000000 00800000:00000004:2.0:1775764768.160712:0:12343:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 490, max_age 500 00000100:00100000:2.0:1775764768.161615:0:12343:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861270656:192.168.202.104@tcp:41:lfs.0 00000100:00100000:2.0:1775764768.161906:0:12343:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c4f08f00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764768.173855:0:12343:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861270656:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764768.173887:0:12343:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:3.0:1775764768.173993:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.174015:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00800000:00000004:3.0:1775764768.174042:0:12343:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 490, max_age 500 00000100:00100000:3.0:1775764768.174086:0:12343:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861270784:192.168.202.104@tcp:41:lfs.0 00000100:00100000:3.0:1775764768.174161:0:12343:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6000 going to sleep for 11 seconds 00010000:00080000:3.0:1775764768.182610:0:12343:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542300 x1862023861270784/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764784 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764768.182648:0:12343:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861270784:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764768.182675:0:12343:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317437/353529 objects 1023743/1024000 00000080:00200080:3.0:1775764768.182747:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184422:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184446:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184451:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184455:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184459:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184463:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184466:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184470:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184473:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184477:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184480:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184512:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184518:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184523:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184526:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184529:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184551:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184556:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184560:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184563:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184566:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184570:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184573:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184576:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184580:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184583:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184585:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184588:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184591:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184594:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184596:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184600:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184602:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184605:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184608:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184611:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184613:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184616:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184618:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184622:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184624:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184646:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184650:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184654:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184657:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184661:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184665:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184668:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184670:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184674:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184677:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184680:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184683:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184686:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184689:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184693:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184696:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184699:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184703:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184706:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184710:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184714:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184717:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184721:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184725:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184728:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184738:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184744:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.184747:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00000080:00200080:3.0:1775764768.184750:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:3.0:1775764768.186786:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7c00 uarg=0000000000000000 00010000:00010000:0.0:1775764768.186935:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200080:2.0:1775764768.188945:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.188974:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.188986:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.188992:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.188997:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189016:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189024:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189028:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189032:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189036:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189039:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189042:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189046:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189049:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189052:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189056:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189059:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189062:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189066:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189069:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189073:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189077:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189081:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189085:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189089:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189093:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189097:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189100:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189104:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189108:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189121:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189126:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189130:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189134:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189137:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189151:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189156:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189160:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189163:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189166:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189170:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189173:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189177:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189180:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189183:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189186:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189189:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189192:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189196:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189199:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189202:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189205:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189208:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00800000:00000080:2.0:1775764768.189211:0:12343:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.189216:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.189230:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00020000:00000004:2.0:1775764768.189239:0:12343:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 490, max_age 500 00000100:00100000:2.0:1775764768.189297:0:12343:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861270912:192.168.202.104@tcp:13:lfs.0 00000100:00100000:2.0:1775764768.189397:0:12343:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c4f08b40 going to sleep for 11 seconds 00010000:00080000:2.0:1775764768.196676:0:12343:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0710a80 x1862023861270912/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764784 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764768.196697:0:12343:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861270912:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764768.196727:0:12343:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901369/958279 objects 261712/262144 00000080:00200080:2.0:1775764768.196810:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.196838:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00020000:00000004:2.0:1775764768.196845:0:12343:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 490, max_age 500 00000100:00100000:2.0:1775764768.196882:0:12343:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861271040:192.168.202.104@tcp:13:lfs.0 00000100:00100000:2.0:1775764768.196952:0:12343:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c4f08b40 going to sleep for 11 seconds 00010000:00080000:2.0:1775764768.205855:0:12343:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0710a80 x1862023861271040/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764784 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764768.205877:0:12343:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12343:1862023861271040:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764768.205905:0:12343:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901369/958279 objects 261712/262144 00000080:00200080:2.0:1775764768.205987:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206004:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206011:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206015:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206020:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206024:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206041:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206046:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206050:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206053:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206057:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206061:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206065:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206068:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206072:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206075:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206078:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206082:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206086:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206089:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206093:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206097:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206102:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206105:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206109:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206112:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206115:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206118:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206122:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206125:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206130:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206152:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206159:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206164:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206168:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206171:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206175:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206179:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206183:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206186:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206189:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206192:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206196:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206198:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206202:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206205:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206209:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206212:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206216:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206219:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206223:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206227:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206231:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206234:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206238:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206241:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206253:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206258:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206262:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206266:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206270:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206273:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206277:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206281:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206284:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206287:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206291:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206295:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206300:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206303:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206308:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206312:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206315:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206319:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206322:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206326:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206329:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206332:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206335:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206339:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206343:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206354:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206360:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206364:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206368:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206371:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206375:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206379:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206383:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206386:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206389:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206392:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206396:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206399:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206402:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206406:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206409:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206413:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206417:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206421:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206425:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206428:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206432:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206435:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206440:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206444:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206456:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206460:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206465:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206469:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206473:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206476:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206480:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206522:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206529:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206533:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206538:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206541:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206546:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206549:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206553:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206557:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206561:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206564:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200080:2.0:1775764768.206568:0:12343:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc4c5b2240 00020000:00000080:2.0:1775764768.206572:0:12343:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132c00 uarg=0000000000000000 00000080:00200008:2.0:1775764768.206587:0:12343:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000100:00100000:1.0:1775764768.621856:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861271168:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764768.621945:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271424:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764768.621964:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271680:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764768.621982:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764768.621985:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271296:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764768.621996:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271552:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764768.636255:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271296:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764768.636277:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0ea00 x1862023861271552/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764768.636288:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271552:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764768.636298:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861271168:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764768.636307:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0e680 x1862023861271424/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764768.636314:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271424:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764768.636324:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0df80 x1862023861271680/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764768.636332:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861271680:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764769.208288:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.225191:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764771.243858:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.270312:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764773.291975:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764773.740978:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861271808:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764773.741090:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272064:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764773.741119:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272320:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764773.741148:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764773.741154:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861271936:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764773.741178:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272192:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764773.753049:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861271936:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764773.753072:0:2403:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-OST0000-osc-ffffa0e9c80cb000 of lustre-OST0000_UUID@192.168.202.104@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1775764773.753078:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1775764773.753082:0:2403:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffffa0e9c80cb000: Connection to lustre-OST0000 (at 192.168.202.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1775764773.790905:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764773.790923:0:2403:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 490 00000100:00080000:3.0:1775764773.790988:0:2403:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764773.791000:0:2403:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1775764773.791043:0:2403:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861272448/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764773.791107:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272192:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764773.791140:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861271808:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764773.791155:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0710700 x1862023861272064/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764789 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764773.791166:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272064:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764773.791177:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0713b80 x1862023861272320/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764789 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764773.791185:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272320:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764773.792259:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861272448:192.168.202.104@tcp:8:ptlrpcd_01_00.0 00000100:00080000:3.0:1775764773.810183:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764773.810202:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764773.810209:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861272448:192.168.202.104@tcp:8:ptlrpcd_01_00.0 00010000:00010000:1.0:1775764774.315615:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764775.340196:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764776.363800:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.387977:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764778.412241:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764778.859786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764778.859792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764778.859797:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764778.859808:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 507 00000100:00080000:3.0:1775764778.859820:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1775764778.859826:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764778.859836:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1775764778.859855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861272960/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764778.859935:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272704:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764778.859968:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861272960:192.168.202.104@tcp:8:kworker.0 00000100:00100000:2.0:1775764778.860025:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861272576:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764778.862633:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861273088:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764778.864112:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861272832:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764778.865049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764778.865058:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764778.865062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861272960:192.168.202.104@tcp:8:kworker.0 00000100:00100000:2.0:1775764778.875413:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861272576:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764778.877691:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0542a00 x1862023861273088/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764778.877713:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861273088:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764778.893708:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0540380 x1862023861272832/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764778.893737:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861272832:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764778.893908:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861272704:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764779.437966:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764780.459860:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764781.488434:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764782.510426:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764783.532187:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764783.979911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764783.979918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775764783.979936:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764783.979958:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 512 00000100:00080000:0.0:1775764783.979970:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1775764783.979976:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764783.979987:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1775764783.980005:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861273600/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764783.980064:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861273344:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764783.980152:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861273728:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764783.980182:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764783.980188:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861273216:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764783.980215:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861273472:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764783.981430:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861273600:192.168.202.104@tcp:8:kworker.0 00000100:00100000:1.0:1775764783.986784:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861273216:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764783.986812:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09dce00 x1862023861273472/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764783.986825:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861273472:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764783.986839:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861273344:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764783.986850:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09df480 x1862023861273728/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764783.986858:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861273728:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764783.990538:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764783.990553:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764783.990558:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861273600:192.168.202.104@tcp:8:kworker.0 00010000:00010000:0.0:1775764784.555966:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764785.579831:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764786.604157:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764787.628027:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764788.653137:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764789.099882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764789.099888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775764789.099893:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764789.099903:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 517 00000100:00080000:2.0:1775764789.099913:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1775764789.099919:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764789.099929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1775764789.099945:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861274240/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764789.099986:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:2.0:1775764789.099990:0:36:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffffa0e9c80cb000: disconnect after 21s idle 00000100:00100000:0.0:1775764789.100293:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861274240:192.168.202.104@tcp:8:kworker.0 00000100:00100000:3.0:1775764789.100993:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861273856:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764789.101206:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861274112:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764789.101236:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764789.101244:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861273984:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764789.117784:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764789.117799:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:0.0:1775764789.117805:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861274240:192.168.202.104@tcp:8:kworker.0 00000100:00100000:3.0:1775764789.117973:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861273984:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764789.118005:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861273856:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764789.118019:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0710380 x1862023861274112/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764805 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764789.118032:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861274112:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764789.121425:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861274368:192.168.202.104@tcp:9:kworker.0 00010000:00080000:0.0:1775764789.132577:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0710000 x1862023861274368/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764789.132615:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9d0710000 x1862023861274368/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764799 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764789.132634:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1775764789.132639:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861274368:192.168.202.104@tcp:9:kworker.0 00010000:00010000:1.0:1775764789.683346:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764790.699878:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764791.734198:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764792.748147:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764792.888868:0:2394:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.104@tcp (source: 12345-192.168.202.104@tcp) 00000100:00100000:0.0:1775764792.889177:0:6162:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862023885553920 00000100:00100000:0.0:1775764792.889192:0:6162:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862023885553920 00000100:00100000:0.0:1775764792.889267:0:6162:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.104@tcp, seq: 5 00000100:00100000:0.0:1775764792.889279:0:6162:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffffa0e9c8d0df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2bf32c54-7e4b-4438-8419-d956344e0e00+3:14120:x1862023885553920:12345-192.168.202.104@tcp:104: 00010000:00010000:0.0:1775764792.889301:0:6162:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764792.889415:0:6162:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764792.889422:0:6162:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffffa0e9c8d0df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2bf32c54-7e4b-4438-8419-d956344e0e00+3:14120:x1862023885553920:12345-192.168.202.104@tcp:104: Request processed in 145us (574us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764792.889431:0:6162:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.104@tcp, seq: 5 00010000:00010000:2.0:1775764792.889522:0:6650:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764792.889532:0:6650:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764792.889541:0:6650:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0e9c0030c00 already unused, calling callback (ffffffffc112fc20) 10000000:00010000:2.0:1775764792.889546:0:6650:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764792.889602:0:6650:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1775764792.889610:0:6650:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764792.889617:0:6650:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1775764792.889680:0:6650:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764792.889687:0:6650:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764792.889722:0:6650:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764792.889730:0:6650:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030c00/0x64300da17553cc2d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76efb9 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764792.889756:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764792.889757:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764792.889759:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764792.890057:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861274496:192.168.202.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764792.899006:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861274496:192.168.202.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764793.772476:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764794.225047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764794.225056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764794.225061:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764794.225076:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 522 00000100:00080000:3.0:1775764794.225090:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764794.225099:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764794.225114:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1775764794.225134:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861275008/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764794.225197:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861274624:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764794.225275:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861275008:192.168.202.104@tcp:8:kworker.0 00000100:00100000:3.0:1775764794.225323:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861274880:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764794.225352:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764794.225357:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861274752:192.168.202.104@tcp:400:kworker.0 00010000:00010000:3.0:1775764794.230126:0:2404:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1775764794.230150:0:6171:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1775764794.230167:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861274752:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764794.230177:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764794.230179:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764794.230180:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764794.230183:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861274624:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764794.230198:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0541880 x1862023861274880/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764810 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764794.230210:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861274880:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764794.230261:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0541500 x1862023861275008/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764819 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764794.230278:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to target with instance 2 00000080:00000004:3.0:1775764794.230301:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764794.230310:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764794.230318:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1775764794.230324:0:2400:0:(import.c:1694:ptlrpc_import_recovery_state_machine()) replay requested by lustre-OST0000_UUID 00000100:00080000:3.0:1775764794.230329:0:2400:0:(recover.c:53:ptlrpc_replay_next()) import ffffa0e9d071e000 from lustre-OST0000_UUID committed 0 last 0 00000100:00080000:3.0:1775764794.230332:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:1.0:1775764794.231288:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861275008:192.168.202.104@tcp:8:kworker.0 00010000:00080000:0.0:1775764794.231814:0:12557:0:(ldlm_request.c:2960:ldlm_lock_replay_thread()) lock replay thread lustre-OST0000-osc-ffffa0e9c80cb000 to lustre-OST0000_UUID@192.168.202.104@tcp 00010000:00010000:0.0:1775764794.231820:0:12557:0:(ldlm_request.c:2876:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-OST0000-osc-ffffa0e9c80cb000 (0) 00010000:00010000:0.0:1775764794.231826:0:12557:0:(ldlm_request.c:2889:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-OST0000-osc-ffffa0e9c80cb000 00000100:00080000:0.0:1775764794.231888:0:12557:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:1.0:1775764794.232432:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861275136:192.168.202.104@tcp:400:ldlm_lock_repla.0 00010000:00080000:1.0:1775764794.279166:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dd500 x1862023861275136/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764810 ref 1 fl Rpc:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764794.279198:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:1.0:1775764794.279205:0:2400:0:(client.c:3204:ptlrpc_resend_req()) @@@ going to resend req@ffffa0e9d09dd500 x1862023861275136/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764810 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764794.279213:0:2400:0:(client.c:3213:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa0e9d09dd500 x1862023861275136/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764810 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764794.279221:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from RECOVER to FULL 00000080:00000004:1.0:1775764794.279235:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:1.0:1775764794.279242:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:02000000:1.0:1775764794.279249:0:2400:0:(import.c:1737:ptlrpc_import_recovery_state_machine()) lustre-OST0000-osc-ffffa0e9c80cb000: Connection restored to 192.168.202.104@tcp (at 192.168.202.104@tcp) 00000100:00100000:1.0:1775764794.287331:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861275136:192.168.202.104@tcp:400:ldlm_lock_repla.0 10000000:01000000:0.0:1775764794.476110:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:0.0:1775764794.476118:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-cliir-ffffa0e9c80cb000 from 12 10000000:01000000:0.0:1775764794.476143:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:0.0:1775764794.476214:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764794.476226:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764794.476238:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764794.476267:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861275264:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764794.476373:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7300 going to sleep for 11 seconds 00000100:00100000:0.0:1775764794.492796:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861275264:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1775764794.492840:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764794.492853:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764794.492935:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861275392:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000100:00100000:0.0:1775764794.493222:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7300 going to sleep for 11 seconds 00000100:00100000:0.0:1775764794.513267:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861275392:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000020:00000080:0.0:1775764794.513325:0:6174:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 10000000:01000000:0.0:1775764794.513466:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:0.0:1775764794.513477:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764794.513754:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764794.798480:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764795.820302:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764796.844148:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764797.867802:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764798.891831:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764799.344557:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861275520:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764799.344712:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861275776:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764799.344754:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764799.344761:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861275648:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764799.344788:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861275904:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764799.352989:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861275648:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764799.353016:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711180 x1862023861275904/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764815 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764799.353050:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861275904:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764799.353066:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0710380 x1862023861275776/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764815 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764799.353075:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861275776:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764799.355959:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861275520:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764799.916417:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764800.940106:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764801.973570:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764802.991174:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764804.011729:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764804.461311:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861276160:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764804.461529:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861276032:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764804.463628:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861276416:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764804.468589:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861276288:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764804.506617:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861276032:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764804.506668:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542680 x1862023861276416/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764820 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764804.506682:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861276416:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764804.510277:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861276160:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764804.510353:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542a00 x1862023861276288/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764820 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764804.510367:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861276288:192.168.202.104@tcp:400:kworker.0 00000080:00200008:0.0:1775764804.997657:0:12663:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764804.997672:0:12663:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764804.997695:0:12663:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764804.997714:0:12663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1775764804.997729:0:12663:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:0.0:1775764804.997737:0:12663:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764804.997744:0:12663:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764804.997747:0:12663:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764804.997754:0:12663:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014bbcb8 00010000:00010000:0.0:1775764804.997757:0:12663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1775764804.997765:0:12663:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200008:0.0:1775764804.999921:0:12663:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1775764804.999931:0:12663:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764804.999961:0:12663:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764804.999977:0:12663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1775764804.999993:0:12663:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:0.0:1775764805.000000:0:12663:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764805.000009:0:12663:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764805.000012:0:12663:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764805.000020:0:12663:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014bbcb8 00010000:00010000:0.0:1775764805.000023:0:12663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1775764805.000032:0:12663:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:0.0:1775764805.000167:0:12663:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764805.000176:0:12663:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764805.000183:0:12663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:0.0:1775764805.000192:0:12663:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:0.0:1775764805.000199:0:12663:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764805.000203:0:12663:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764805.000205:0:12663:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764805.000209:0:12663:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014bbbb0 00010000:00010000:0.0:1775764805.000218:0:12663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:0.0:1775764805.000226:0:12663:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:0.0:1775764805.000235:0:12663:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:0.0:1775764805.000258:0:12663:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:0.0:1775764805.002063:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:0.0:1775764805.002091:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75800 uarg=0000000000000000 00800000:00000004:0.0:1775764805.002100:0:12663:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 501, max_age 537 00000100:00100000:0.0:1775764805.002151:0:12663:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12663:1862023861276544:192.168.202.104@tcp:41:lfs.0 00000100:00100000:0.0:1775764805.002747:0:12663:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764805.011523:0:12663:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12663:1862023861276544:192.168.202.104@tcp:41:lfs.0 00800000:00000004:2.0:1775764805.011552:0:12663:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:2.0:1775764805.011643:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.011664:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00800000:00000004:2.0:1775764805.011673:0:12663:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 501, max_age 537 00000100:00100000:2.0:1775764805.011714:0:12663:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12663:1862023861276672:192.168.202.104@tcp:41:lfs.0 00000100:00100000:2.0:1775764805.011807:0:12663:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d8600 going to sleep for 11 seconds 00010000:00080000:2.0:1775764805.023262:0:12663:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0712300 x1862023861276672/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764821 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764805.025210:0:12663:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12663:1862023861276672:192.168.202.104@tcp:41:lfs.0 00800000:00000004:2.0:1775764805.025246:0:12663:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317437/353529 objects 1023743/1024000 00000080:00200080:2.0:1775764805.025771:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025789:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025797:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025801:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025806:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025810:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025814:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025818:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025822:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025825:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025829:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025833:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025856:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025861:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025866:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025869:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025872:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025875:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025880:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025883:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025886:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025889:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025893:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025896:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025900:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025903:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025906:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025909:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025913:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025916:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025920:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025924:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025928:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025931:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025935:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025937:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025941:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025952:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025958:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025963:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025967:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025971:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025974:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025978:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025983:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025986:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025990:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.025993:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.025997:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026000:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026005:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026008:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026013:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026017:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026021:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026025:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026029:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026033:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026037:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026041:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026045:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026048:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026059:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026063:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026066:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026069:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026072:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026075:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026079:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026083:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026087:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026090:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026094:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026098:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026102:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026106:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026111:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026114:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026118:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026121:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026125:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026129:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026133:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026136:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026140:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026143:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026147:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026158:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026164:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026167:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026171:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026174:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026178:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026181:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026185:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026189:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026192:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026196:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026200:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026203:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026207:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026210:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026214:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026218:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026222:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026225:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026228:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026231:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026234:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026237:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026240:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026243:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026252:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026255:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.026259:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.026261:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.027391:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.027415:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.027430:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.027435:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.027439:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.027443:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.027446:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.027450:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.027454:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00800000:00000080:2.0:1775764805.027457:0:12663:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.027462:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.027469:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00020000:00000004:2.0:1775764805.027559:0:12663:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 501, max_age 537 00000100:00100000:2.0:1775764805.027614:0:12663:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12663:1862023861276800:192.168.202.104@tcp:13:lfs.0 00000100:00100000:2.0:1775764805.027708:0:12663:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d8600 going to sleep for 11 seconds 00010000:00080000:2.0:1775764805.031819:0:12663:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0712300 x1862023861276800/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764821 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764805.031838:0:12663:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12663:1862023861276800:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764805.031864:0:12663:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:2.0:1775764805.031927:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.031956:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00020000:00000004:2.0:1775764805.031963:0:12663:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 501, max_age 537 00000100:00080000:2.0:1775764805.031974:0:12663:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 490 00000100:00080000:2.0:1775764805.031984:0:12663:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764805.031994:0:12663:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8437760, lost: 0. 00000100:00100000:2.0:1775764805.032014:0:12663:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861276928/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764805.032058:0:12663:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa0e9d0712300 x1862023861277056/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764805.032067:0:12663:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d8600 going to sleep for 0 seconds 00000100:00100000:3.0:1775764805.032799:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861276928:192.168.202.104@tcp:8:lfs.0 00010000:00010000:0.0:1775764805.038307:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:3.0:1775764805.043417:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0712d80 x1862023861276928/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764815 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764805.043435:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffffa0e9c80cb000: committing for last_committed 0 gen 4 00000100:00080000:3.0:1775764805.043446:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to target with instance 1 00000100:00080000:3.0:1775764805.043457:0:2400:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffffa0e9c80cb000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:3.0:1775764805.043530:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764805.043538:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764805.043544:0:2400:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:3.0:1775764805.043548:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:3.0:1775764805.043555:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764805.043557:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764805.043566:0:2400:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000002723f87c): req@ffffa0e9d0712300 x1862023861277056/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764805.043591:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861276928:192.168.202.104@tcp:8:lfs.0 00000100:00100000:2.0:1775764805.043820:0:12663:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d8600 going to sleep for 11 seconds 00010000:00080000:2.0:1775764805.050885:0:12663:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0712300 x1862023861277056/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764821 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764805.050907:0:12663:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:12663:1862023861277056:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764805.050932:0:12663:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901369/958279 objects 261712/262144 00000080:00200080:2.0:1775764805.051010:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051027:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051034:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051039:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051043:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051046:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051051:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051054:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051058:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051077:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051082:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051086:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051089:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051092:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051096:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051099:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051103:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051107:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051111:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051115:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051119:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051122:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051126:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051130:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051133:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051137:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051140:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051144:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051147:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051150:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051154:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051158:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051162:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051165:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051177:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051181:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051185:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051189:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051193:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051196:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051200:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051203:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051206:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051209:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051213:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051216:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051220:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051223:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051227:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051230:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051233:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051236:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051239:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051242:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051246:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051249:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051253:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051256:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051260:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051284:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051291:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051295:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051299:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051303:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051307:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051311:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051315:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051319:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051323:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051327:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051331:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051335:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051339:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051342:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051346:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051350:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051354:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051358:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051361:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051365:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051369:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051372:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051376:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051380:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051392:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051397:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051400:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051404:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051408:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051411:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051426:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051431:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051435:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051439:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051443:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051446:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051450:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051452:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051455:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051459:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051463:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051466:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.051470:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.051473:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052796:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052814:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052823:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052827:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052831:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052846:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052851:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052854:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052858:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052862:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052865:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052869:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052873:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052876:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052880:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052883:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052887:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052890:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052894:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052897:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200080:2.0:1775764805.052901:0:12663:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe2025ebb0 00020000:00000080:2.0:1775764805.052905:0:12663:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8131000 uarg=0000000000000000 00000080:00200008:2.0:1775764805.052919:0:12663:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:0.0:1775764806.060064:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764807.084756:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764808.114993:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764809.132304:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764809.582583:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277312:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764809.582750:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277568:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764809.582787:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:0.0:1775764809.582792:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861277184:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764809.582820:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277440:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764809.582841:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277696:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764809.592265:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09df480 x1862023861277568/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764825 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764809.592323:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277568:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764809.594262:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861277184:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764809.594291:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dca80 x1862023861277696/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764825 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764809.594305:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277696:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764809.599281:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09de680 x1862023861277440/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764825 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764809.599304:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277440:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764809.599320:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861277312:192.168.202.104@tcp:400:kworker.0 00000001:02000400:3.0:1775764809.628623:0:12916:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg204-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid 00010000:00010000:0.0:1775764810.156871:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764811.187403:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764811.599070:0:13073:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid in FULL state after 0 sec 00010000:00010000:0.0:1775764812.204099:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764813.234788:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1775764813.891480:0:13111:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764813.893036:0:13111:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764813.893065:0:13111:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764813.893085:0:13111:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:0.0:1775764813.893104:0:13111:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:0.0:1775764813.893113:0:13111:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764813.893121:0:13111:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764813.893124:0:13111:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764813.893131:0:13111:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d7fcb8 00010000:00010000:0.0:1775764813.893135:0:13111:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:0.0:1775764813.893144:0:13111:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200008:0.0:1775764813.894462:0:13111:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1775764813.894470:0:13111:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764813.894643:0:13111:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764813.894659:0:13111:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:0.0:1775764813.894673:0:13111:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:0.0:1775764813.894679:0:13111:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764813.894685:0:13111:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764813.894688:0:13111:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764813.894693:0:13111:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d7fcb8 00010000:00010000:0.0:1775764813.894697:0:13111:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:0.0:1775764813.894704:0:13111:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:0.0:1775764813.894832:0:13111:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764813.895062:0:13111:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764813.895071:0:13111:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:0.0:1775764813.895079:0:13111:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:0.0:1775764813.895085:0:13111:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764813.895089:0:13111:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764813.895092:0:13111:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764813.895095:0:13111:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d7fbb0 00010000:00010000:0.0:1775764813.895105:0:13111:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:0.0:1775764813.895112:0:13111:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:0.0:1775764813.895121:0:13111:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:0.0:1775764813.895129:0:13111:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:0.0:1775764813.895294:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.895311:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00800000:00000004:0.0:1775764813.895318:0:13111:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 538, max_age 546 00000100:00100000:0.0:1775764813.895365:0:13111:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861277824:192.168.202.104@tcp:41:lfs.0 00000100:00100000:0.0:1775764813.895514:0:13111:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7f00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764813.901069:0:13111:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861277824:192.168.202.104@tcp:41:lfs.0 00800000:00000004:0.0:1775764813.901096:0:13111:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:0.0:1775764813.901237:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.901255:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00800000:00000004:0.0:1775764813.901273:0:13111:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 538, max_age 546 00000100:00100000:0.0:1775764813.901307:0:13111:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861277952:192.168.202.104@tcp:41:lfs.0 00000100:00100000:0.0:1775764813.901376:0:13111:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7f00 going to sleep for 11 seconds 00010000:00080000:0.0:1775764813.913799:0:13111:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09df480 x1862023861277952/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764829 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764813.913820:0:13111:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861277952:192.168.202.104@tcp:41:lfs.0 00800000:00000004:0.0:1775764813.913843:0:13111:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317437/353529 objects 1023743/1024000 00000080:00200080:0.0:1775764813.913921:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.913952:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.913961:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.913965:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.913970:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.913974:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.913978:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.913982:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.913986:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.913989:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.913993:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.913997:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914001:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914004:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914008:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914012:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914016:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914019:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914023:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914027:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914030:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914033:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914037:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914040:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914044:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914047:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914058:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914063:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914067:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914071:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914075:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914078:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914081:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914084:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914088:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914091:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914096:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914099:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914103:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914107:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914110:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914114:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914117:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914121:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914124:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914127:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914131:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914134:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914138:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914141:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914145:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914155:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914160:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914164:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914168:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914172:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914175:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914178:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914182:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914184:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914188:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914191:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914194:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914196:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914199:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914202:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914205:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914209:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914213:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914216:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914220:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914223:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914227:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914230:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914233:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914236:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914247:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914251:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914255:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914259:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914262:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914266:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914270:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914274:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914278:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914281:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914285:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914288:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914292:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914295:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914299:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914302:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914306:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914309:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914314:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914317:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914321:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914324:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914328:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914331:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914335:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914347:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914352:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914356:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914360:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914363:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914367:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914370:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914373:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914377:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914380:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914384:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914387:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914391:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914407:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914411:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914416:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914420:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914424:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914428:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914432:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914435:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914438:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914442:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914445:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00800000:00000080:0.0:1775764813.914448:0:13111:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.914462:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.914469:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00020000:00000004:0.0:1775764813.914477:0:13111:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 538, max_age 546 00000100:00100000:0.0:1775764813.914563:0:13111:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861278080:192.168.202.104@tcp:13:lfs.0 00000100:00100000:0.0:1775764813.914622:0:13111:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7f00 going to sleep for 11 seconds 00010000:00080000:0.0:1775764813.939246:0:13111:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09df480 x1862023861278080/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764829 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764813.939278:0:13111:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861278080:192.168.202.104@tcp:13:lfs.0 00020000:00000004:0.0:1775764813.939304:0:13111:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:0.0:1775764813.939452:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.939470:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00020000:00000004:0.0:1775764813.939478:0:13111:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 538, max_age 546 00000100:00100000:0.0:1775764813.939550:0:13111:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861278208:192.168.202.104@tcp:13:lfs.0 00000100:00100000:0.0:1775764813.939614:0:13111:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7f00 going to sleep for 11 seconds 00010000:00080000:0.0:1775764813.957606:0:13111:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09df480 x1862023861278208/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764829 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764813.957625:0:13111:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13111:1862023861278208:192.168.202.104@tcp:13:lfs.0 00020000:00000004:0.0:1775764813.957650:0:13111:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901369/958279 objects 261712/262144 00000080:00200080:0.0:1775764813.957732:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957754:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957761:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957765:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957768:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957788:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957794:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957797:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957801:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957804:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957808:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957812:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957815:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957818:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957821:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957824:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957827:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957830:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957834:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957837:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957839:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957842:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957845:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957848:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957852:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957856:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957859:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957863:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957867:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957870:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957883:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957888:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957892:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957895:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957899:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957902:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957906:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957909:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957913:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957915:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957919:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957922:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957926:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957929:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957933:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957935:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957939:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957941:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957945:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957948:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957952:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957955:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957959:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957962:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957965:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957975:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957981:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957985:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957989:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.957993:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.957996:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958000:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958003:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958006:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958010:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958013:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958016:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958019:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958022:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958025:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958029:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958032:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958036:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958039:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958043:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958046:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958049:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958053:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958057:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958060:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958072:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958077:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958081:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958083:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958087:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958090:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958093:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958096:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958100:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958102:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958106:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958109:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958113:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958116:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958120:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958123:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958127:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958131:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958135:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958138:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958142:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958145:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958148:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958151:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958154:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958166:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958170:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958174:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958177:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958181:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958184:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958187:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958191:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958194:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958198:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958201:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958205:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958208:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958212:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958215:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958219:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958222:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958225:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958228:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764813.958231:0:13111:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffe4557ccc0 00020000:00000080:0.0:1775764813.958234:0:13111:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200008:0.0:1775764813.958248:0:13111:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:0.0:1775764814.254761:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764814.699930:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278464:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764814.700034:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278720:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764814.700068:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1775764814.700073:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861278336:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764814.700100:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278592:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764814.700122:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278848:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764814.705755:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861278336:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764814.705776:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0ea00 x1862023861278592/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764830 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764814.705802:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278592:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764814.706531:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8defb80 x1862023861278848/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764830 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764814.706546:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278848:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764814.706564:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278464:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764814.706575:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8d0d500 x1862023861278720/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764830 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764814.706585:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8d0d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861278720:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764815.277411:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764816.304023:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764817.324247:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764818.348577:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764819.380715:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764819.820464:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279104:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764819.820657:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279360:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764819.820694:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1775764819.820700:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861278976:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764819.820756:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279232:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764819.820782:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279488:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764819.832758:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861278976:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764819.832791:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0710700 x1862023861279232/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764835 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764819.832805:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279232:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764819.832819:0:2404:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-OST0001-osc-ffffa0e9c80cb000 of lustre-OST0001_UUID@192.168.202.104@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1775764819.832826:0:2404:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764819.832830:0:2404:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffffa0e9c80cb000: Connection to lustre-OST0001 (at 192.168.202.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764819.856867:0:2404:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764819.856893:0:2404:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 538 00000100:00080000:2.0:1775764819.856920:0:2404:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764819.856937:0:2404:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1775764819.856986:0:2404:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861279616/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764819.857113:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861279616:192.168.202.104@tcp:8:ptlrpcd_01_01.0 00000100:00100000:2.0:1775764819.858560:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279488:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764819.859337:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279104:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764819.859913:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0710a80 x1862023861279360/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764835 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764819.859956:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279360:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764819.866189:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764819.866208:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:0.0:1775764819.866215:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861279616:192.168.202.104@tcp:8:ptlrpcd_01_01.0 00010000:00010000:0.0:1775764820.396784:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764821.421804:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764822.449357:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764823.467956:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764824.493317:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764824.941407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764824.941414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775764824.941419:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764824.941435:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 553 00000100:00080000:2.0:1775764824.941452:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1775764824.941460:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764824.941476:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1775764824.941526:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861280256/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764824.941676:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861279744:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764824.941817:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861280000:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764824.941858:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1775764824.941865:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279872:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764824.941891:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861280128:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764824.942376:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861280256:192.168.202.104@tcp:8:kworker.0 00010000:00080000:3.0:1775764824.946583:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0710700 x1862023861280128/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764840 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764824.946611:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861280128:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764824.946626:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861279744:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764824.953346:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861279872:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764824.953398:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0710e00 x1862023861280000/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764840 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764824.953411:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861280000:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764824.962143:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764824.962163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:0.0:1775764824.962199:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861280256:192.168.202.104@tcp:8:kworker.0 00010000:00010000:2.0:1775764825.516118:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764826.539977:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764827.563728:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764828.588052:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764829.613606:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764830.063450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764830.063458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775764830.063463:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764830.063478:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 558 00000100:00080000:3.0:1775764830.063544:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1775764830.063552:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764830.063568:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1775764830.063590:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861280896/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764830.063640:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861280512:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764830.063787:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861280768:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764830.063826:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764830.063832:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861280384:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764830.063862:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861280640:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764830.064015:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861280896:192.168.202.104@tcp:8:kworker.0 00000100:00100000:3.0:1775764830.075121:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861280384:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764830.075153:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0542a00 x1862023861280768/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764846 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764830.075182:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861280768:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764830.080115:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861280512:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764830.081590:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764830.081608:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764830.081613:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861280896:192.168.202.104@tcp:8:kworker.0 00010000:00080000:3.0:1775764830.081666:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d0541880 x1862023861280640/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764846 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764830.081680:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861280640:192.168.202.104@tcp:400:kworker.0 00010000:00010000:0.0:1775764830.635935:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764831.659673:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764832.683718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764833.707790:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764834.733954:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764835.183807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:0.0:1775764835.183813:0:37:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffffa0e9c80cb000: disconnect after 22s idle 00000100:00100000:1.0:1775764835.184682:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861281024:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764835.184795:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861281280:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764835.184835:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764835.184842:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861281152:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764835.190418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764835.190425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775764835.190430:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764835.190440:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 563 00000100:00080000:0.0:1775764835.190454:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1775764835.190462:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764835.190571:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1775764835.190582:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861281408:192.168.202.104@tcp:9:kworker.0 00000100:00100000:0.0:1775764835.190620:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861281536/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764835.190743:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861281536:192.168.202.104@tcp:8:kworker.0 00000100:00100000:0.0:1775764835.207577:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861281152:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764835.207595:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861281024:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764835.207610:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09dc700 x1862023861281280/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764851 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764835.207662:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861281280:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764835.207759:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764835.207771:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764835.207776:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861281536:192.168.202.104@tcp:8:kworker.0 00010000:00080000:3.0:1775764835.207796:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa0e9d09dd180 x1862023861281408/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764845 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764835.207811:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9d09dd180 x1862023861281408/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764845 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764835.207820:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1775764835.207823:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861281408:192.168.202.104@tcp:9:kworker.0 00010000:00010000:1.0:1775764835.755830:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764836.693899:0:2393:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.104@tcp (source: 12345-192.168.202.104@tcp) 00000100:00100000:0.0:1775764836.694115:0:6161:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764836.694150:0:6161:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862023885578112 00000100:00100000:0.0:1775764836.694163:0:6161:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862023885578112 00000100:00100000:0.0:1775764836.694241:0:6161:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.104@tcp, seq: 6 00000100:00100000:0.0:1775764836.694253:0:6161:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffffa0e9c52d2050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:14120:x1862023885578112:12345-192.168.202.104@tcp:104: 00010000:00010000:0.0:1775764836.694276:0:6161:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764836.694390:0:6161:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764836.694397:0:6161:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffffa0e9c52d2050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:2bf32c54-7e4b-4438-8419-d956344e0e00+3:14120:x1862023885578112:12345-192.168.202.104@tcp:104: Request processed in 147us (527us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764836.694406:0:6161:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.104@tcp, seq: 6 00010000:00010000:2.0:1775764836.694686:0:6171:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764836.694695:0:6171:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.694704:0:6171:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0e9c0030e00 already unused, calling callback (ffffffffc112fc20) 10000000:00010000:2.0:1775764836.694708:0:6171:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.694718:0:6171:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1775764836.694725:0:6171:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764836.694751:0:6171:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1775764836.694812:0:6171:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.694819:0:6171:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764836.694847:0:6171:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.694855:0:6171:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030e00/0x64300da17553cc34 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x39b1144a2f76f084 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.694876:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764836.694878:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764836.694879:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764836.694903:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861281664:192.168.202.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764836.703546:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861281664:192.168.202.104@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764836.780721:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764837.803824:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1775764837.806308:0:6174:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:0.0:1775764837.806318:0:6174:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-cliir-ffffa0e9c80cb000 from 13 10000000:01000000:0.0:1775764837.806325:0:6174:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:0.0:1775764837.806396:0:6174:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030600/0x64300da17553cc3b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764837.806412:0:6174:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030600/0x64300da17553cc3b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764837.806425:0:6174:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030600/0x64300da17553cc3b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764837.806461:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861281792:192.168.202.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764837.806617:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d70c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764837.811583:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861281792:192.168.202.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1775764837.811623:0:6174:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030600/0x64300da17553cc3b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76f11e expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.811640:0:6174:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030600/0x64300da17553cc3b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x39b1144a2f76f11e expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764837.811742:0:6174:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861281920:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1775764837.813444:0:6174:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d60c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764837.823800:0:6174:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:2bf32c54-7e4b-4438-8419-d956344e0e00:6174:1862023861281920:192.168.202.104@tcp:256:ll_cfg_requeue.0 00000020:00000080:3.0:1775764837.823868:0:6174:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 10000000:01000000:3.0:1775764837.825316:0:6174:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.104@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1775764837.825334:0:6174:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030600/0x64300da17553cc3b lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76f11e expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.825350:0:6174:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.104@tcp lock: ffffa0e9c0030600/0x64300da17553cc3b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x39b1144a2f76f11e expref: -99 pid: 6174 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:2.0:1775764837.864212:0:13342:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffffa0e9d07fd800 already not connected (conn 8, was 0): DISCONN 00010000:00080000:2.0:1775764837.864253:0:13342:0:(ldlm_lib.c:85:import_set_conn()) imp ffffa0e9d07fd800@lustre-OST0001-osc-ffffa0e9c80cb000: found existing conn 192.168.202.104@tcp, moved to head 00000100:00080000:2.0:1775764837.864261:0:13342:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764837.864273:0:13342:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 0 00000100:00080000:2.0:1775764837.864285:0:13342:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffffa0e9c80cb000: import ffffa0e9d07fd800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764837.864300:0:13342:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1775764837.864350:0:13342:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713800 x1862023861282048/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764837.870818:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861282048:192.168.202.104@tcp:8:lctl.0 00010000:00080000:3.0:1775764837.898776:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713800 x1862023861282048/t0(0) o8->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764847 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lctl.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764837.898805:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffffa0e9c80cb000: connect to target with instance 2 00000080:00000004:3.0:1775764837.898828:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764837.898837:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764837.898846:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1775764837.898864:0:2400:0:(import.c:1694:ptlrpc_import_recovery_state_machine()) replay requested by lustre-OST0001_UUID 00000100:00080000:3.0:1775764837.898868:0:2400:0:(recover.c:53:ptlrpc_replay_next()) import ffffa0e9d07fd800 from lustre-OST0001_UUID committed 0 last 0 00000100:00080000:3.0:1775764837.898871:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:3.0:1775764837.899322:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861282048:192.168.202.104@tcp:8:lctl.0 00010000:00080000:0.0:1775764837.899463:0:13343:0:(ldlm_request.c:2960:ldlm_lock_replay_thread()) lock replay thread lustre-OST0001-osc-ffffa0e9c80cb000 to lustre-OST0001_UUID@192.168.202.104@tcp 00010000:00010000:0.0:1775764837.899469:0:13343:0:(ldlm_request.c:2876:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-OST0001-osc-ffffa0e9c80cb000 (0) 00010000:00010000:0.0:1775764837.899476:0:13343:0:(ldlm_request.c:2889:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-OST0001-osc-ffffa0e9c80cb000 00000100:00080000:0.0:1775764837.899571:0:13343:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:3.0:1775764837.899794:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861282176:192.168.202.104@tcp:400:ldlm_lock_repla.0 00010000:00080000:3.0:1775764838.445764:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09df800 x1862023861282176/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764853 ref 1 fl Rpc:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764838.445792:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:3.0:1775764838.445798:0:2400:0:(client.c:3204:ptlrpc_resend_req()) @@@ going to resend req@ffffa0e9d09df800 x1862023861282176/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764853 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764838.445807:0:2400:0:(client.c:3213:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa0e9d09df800 x1862023861282176/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764853 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764838.445815:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from RECOVER to FULL 00000080:00000004:3.0:1775764838.445831:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764838.445839:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:02000000:3.0:1775764838.445847:0:2400:0:(import.c:1737:ptlrpc_import_recovery_state_machine()) lustre-OST0001-osc-ffffa0e9c80cb000: Connection restored to 192.168.202.104@tcp (at 192.168.202.104@tcp) 00000100:00100000:3.0:1775764838.470834:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861282176:192.168.202.104@tcp:400:ldlm_lock_repla.0 00010000:00010000:2.0:1775764838.827837:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764839.854627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764840.299897:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861282432:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764840.300070:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861282688:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764840.300109:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764840.300115:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861282304:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764840.300143:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861282560:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764840.310469:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861282304:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764840.310559:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713480 x1862023861282560/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764856 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764840.310572:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861282560:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764840.310620:0:2404:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1775764840.310626:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861282432:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764840.310640:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711500 x1862023861282688/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764856 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764840.310648:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861282688:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764840.312203:0:6172:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764840.312233:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764840.312235:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764840.312237:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764840.879646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764841.899944:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764842.926312:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764843.949099:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764844.972097:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764845.419944:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861282816:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764845.420148:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861283072:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764845.420215:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1775764845.420222:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861282944:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764845.420258:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861283200:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764845.438404:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861282816:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764845.438439:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542680 x1862023861283072/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764861 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764845.438454:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861283072:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764845.438517:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861282944:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764845.438531:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0542a00 x1862023861283200/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764861 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764845.438539:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861283200:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764845.997156:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764847.019851:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764848.046027:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1775764848.366609:0:13446:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764848.366628:0:13446:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764848.366669:0:13446:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764848.366695:0:13446:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1775764848.366747:0:13446:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:3.0:1775764848.366757:0:13446:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764848.366770:0:13446:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764848.366774:0:13446:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:3.0:1775764848.366784:0:13446:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014cbcb8 00010000:00010000:3.0:1775764848.366789:0:13446:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1775764848.366800:0:13446:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200008:3.0:1775764848.369442:0:13446:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1775764848.369456:0:13446:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764848.370094:0:13446:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764848.370124:0:13446:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1775764848.370143:0:13446:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:3.0:1775764848.370152:0:13446:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764848.370163:0:13446:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764848.370167:0:13446:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:3.0:1775764848.370176:0:13446:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014cbcb8 00010000:00010000:3.0:1775764848.370189:0:13446:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1775764848.370199:0:13446:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:3.0:1775764848.370378:0:13446:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:3.0:1775764848.370388:0:13446:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764848.370396:0:13446:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1775764848.370405:0:13446:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:3.0:1775764848.370412:0:13446:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764848.370416:0:13446:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:3.0:1775764848.370418:0:13446:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:3.0:1775764848.370422:0:13446:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014cbbb0 00010000:00010000:3.0:1775764848.370424:0:13446:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1775764848.370432:0:13446:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:3.0:1775764848.370440:0:13446:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:3.0:1775764848.370450:0:13446:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:3.0:1775764848.370751:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:3.0:1775764848.370772:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00800000:00000004:3.0:1775764848.370787:0:13446:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 547, max_age 580 00000100:00100000:3.0:1775764848.370850:0:13446:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13446:1862023861283328:192.168.202.104@tcp:41:lfs.0 00000100:00100000:3.0:1775764848.371004:0:13446:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9caf6d0c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764848.374085:0:13446:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13446:1862023861283328:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764848.374121:0:13446:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:2.0:1775764848.377196:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.377231:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00800000:00000004:2.0:1775764848.377242:0:13446:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 547, max_age 580 00000100:00100000:2.0:1775764848.377309:0:13446:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13446:1862023861283456:192.168.202.104@tcp:41:lfs.0 00000100:00100000:2.0:1775764848.377467:0:13446:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c980f600 going to sleep for 11 seconds 00010000:00080000:2.0:1775764848.385962:0:13446:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711500 x1862023861283456/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764864 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764848.385990:0:13446:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13446:1862023861283456:192.168.202.104@tcp:41:lfs.0 00800000:00000004:2.0:1775764848.386175:0:13446:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317437/353529 objects 1023743/1024000 00000080:00200080:2.0:1775764848.386382:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386408:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386416:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386420:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386424:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386428:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386431:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386434:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386437:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386441:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386444:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386447:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386451:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386454:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386458:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386461:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386464:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386468:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386471:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386474:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386477:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386480:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386536:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386542:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386547:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386558:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386564:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386567:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386571:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386574:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386578:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386582:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386585:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386588:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386592:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386595:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386598:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386601:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386605:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386608:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386611:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386615:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386618:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386621:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386625:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386628:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386632:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386635:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386639:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386642:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386652:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386656:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386660:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386663:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386667:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386670:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386674:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386677:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386681:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386685:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386689:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386692:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386696:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386700:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386703:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386706:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386710:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386713:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386730:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386734:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386738:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386741:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386744:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386747:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386750:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386760:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386765:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386768:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386771:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386775:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386779:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386782:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386786:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386789:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386793:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386796:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386800:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386803:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386806:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386809:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386813:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386816:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386820:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386823:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386827:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386830:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386834:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386837:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386840:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386843:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386854:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386857:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386860:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386864:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386867:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386870:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386873:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386875:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386887:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386891:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386895:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386898:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386902:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386904:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386908:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386911:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386914:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386917:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386920:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386922:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386925:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386928:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386932:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386934:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386937:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00800000:00000080:2.0:1775764848.386946:0:13446:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00000080:00200080:2.0:1775764848.386950:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:2.0:1775764848.386957:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00020000:00000004:2.0:1775764848.386966:0:13446:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 547, max_age 581 00000100:00080000:2.0:1775764848.386985:0:13446:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 527 00000100:00080000:2.0:1775764848.387000:0:13446:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffffa0e9c80cb000: import ffffa0e9d071e000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764848.387022:0:13446:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1775764848.387051:0:13446:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861283584/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764848.387107:0:13446:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa0e9d0711500 x1862023861283712/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764848.387117:0:13446:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c980f600 going to sleep for 0 seconds 00000100:00100000:3.0:1775764848.387550:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861283584:192.168.202.104@tcp:8:lfs.0 00010000:00080000:3.0:1775764848.400000:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0712680 x1862023861283584/t0(0) o8->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 520/416 e 0 to 0 dl 1775764858 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764848.400030:0:2400:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffffa0e9c80cb000: committing for last_committed 0 gen 4 00000100:00080000:3.0:1775764848.400042:0:2400:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffffa0e9c80cb000: connect to target with instance 2 00000100:00080000:3.0:1775764848.400054:0:2400:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffffa0e9c80cb000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:3.0:1775764848.400068:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764848.400083:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764848.400091:0:2400:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:3.0:1775764848.400095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:3.0:1775764848.400101:0:2400:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1775764848.400104:0:2400:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1775764848.400113:0:2400:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 0000000009fe472c): req@ffffa0e9d0711500 x1862023861283712/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764848.400141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861283584:192.168.202.104@tcp:8:lfs.0 00000100:00100000:2.0:1775764848.400425:0:13446:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c980f600 going to sleep for 11 seconds 00010000:00080000:2.0:1775764848.403219:0:13446:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711500 x1862023861283712/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764864 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764848.403242:0:13446:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13446:1862023861283712:192.168.202.104@tcp:13:lfs.0 00020000:00000004:2.0:1775764848.403272:0:13446:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:2.0:1775764848.403364:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:2.0:1775764848.403386:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8132400 uarg=0000000000000000 00020000:00000004:2.0:1775764848.403395:0:13446:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 547, max_age 581 00000100:00100000:2.0:1775764848.403440:0:13446:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13446:1862023861283840:192.168.202.104@tcp:13:lfs.0 00000100:00100000:2.0:1775764848.403569:0:13446:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c980f600 going to sleep for 11 seconds 00010000:00080000:3.0:1775764848.409688:0:13446:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711500 x1862023861283840/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764864 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764848.409843:0:13446:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13446:1862023861283840:192.168.202.104@tcp:13:lfs.0 00020000:00000004:3.0:1775764848.409996:0:13446:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:3.0:1775764848.410250:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410275:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410287:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410292:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410297:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410301:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410305:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410309:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410313:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410316:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410320:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410324:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410328:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410332:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410337:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410340:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410344:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410347:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410351:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410355:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410359:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410363:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410368:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410372:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410388:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410392:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410396:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410399:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410403:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410406:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410409:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410412:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410416:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410419:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410423:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410425:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410429:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410432:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410436:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410439:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410443:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410447:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410450:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410453:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410457:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410461:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410464:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410468:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410471:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410482:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410520:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410526:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410530:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410534:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410538:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410541:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410545:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410548:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410551:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410555:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410559:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410562:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410566:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410569:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410573:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410576:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410580:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410583:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410587:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410590:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410594:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410598:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410602:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410605:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410617:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410622:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410627:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410631:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410635:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410638:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410642:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410646:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410650:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410653:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410657:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410661:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410666:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410669:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410673:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410677:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410681:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410685:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410689:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410693:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410697:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410701:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410705:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410708:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410713:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410813:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410821:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410825:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410829:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410832:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410837:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410841:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410845:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410849:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410853:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410857:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410861:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410864:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410868:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410872:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410876:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410892:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410898:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410902:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410906:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410909:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410913:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410917:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410921:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410925:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200080:3.0:1775764848.410937:0:13446:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffd3d3d0ce0 00020000:00000080:3.0:1775764848.410942:0:13446:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c7400 uarg=0000000000000000 00000080:00200008:3.0:1775764848.410960:0:13446:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:1.0:1775764849.068062:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764850.093648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764850.540291:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861283968:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764850.540432:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284224:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764850.540478:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284480:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764850.540550:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1775764850.540556:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284096:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764850.540581:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284352:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764850.555944:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284096:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764850.555965:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861283968:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764850.555982:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711880 x1862023861284224/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764866 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764850.556007:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284224:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764850.557724:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0712a00 x1862023861284352/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764866 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764850.557744:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284352:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764850.557759:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711500 x1862023861284480/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764866 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764850.557768:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284480:192.168.202.104@tcp:400:kworker.0 00010000:00010000:3.0:1775764851.117258:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764852.140669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764853.178969:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1775764853.275222:0:13699:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg204-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0001-osc-[-0-9a-f]*.ost_server_uuid 00010000:00010000:3.0:1775764854.190035:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764855.014090:0:13855:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: osc.lustre-OST0001-osc-[-0-9a-f]*.ost_server_uuid in FULL state after 0 sec 00010000:00010000:3.0:1775764855.212328:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764855.660384:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284736:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764855.660526:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284992:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764855.660563:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1775764855.660587:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861284608:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764855.660619:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284864:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764855.660642:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861285120:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764855.666366:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861284608:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764855.666391:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713480 x1862023861284992/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764871 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764855.666405:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284992:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764855.676119:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284736:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764855.689282:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0711880 x1862023861284864/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764871 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764855.689304:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861284864:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764855.689318:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0713100 x1862023861285120/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764871 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764855.689327:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861285120:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764856.235807:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764857.260138:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1775764857.450274:0:13894:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764857.450291:0:13894:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764857.450315:0:13894:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764857.450335:0:13894:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:0.0:1775764857.450351:0:13894:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:0.0:1775764857.450361:0:13894:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764857.450368:0:13894:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764857.450372:0:13894:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764857.450379:0:13894:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d4bcb8 00010000:00010000:0.0:1775764857.450383:0:13894:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:0.0:1775764857.450393:0:13894:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200008:0.0:1775764857.452335:0:13894:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1775764857.452350:0:13894:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764857.452373:0:13894:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764857.452391:0:13894:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1775764857.452406:0:13894:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1775764857.452414:0:13894:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764857.452421:0:13894:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764857.452425:0:13894:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764857.452431:0:13894:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d4bcb8 00010000:00010000:0.0:1775764857.452435:0:13894:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1775764857.452444:0:13894:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:0.0:1775764857.453047:0:13894:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:0.0:1775764857.453060:0:13894:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764857.453069:0:13894:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:0.0:1775764857.453079:0:13894:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:0.0:1775764857.453085:0:13894:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764857.453089:0:13894:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:0.0:1775764857.453100:0:13894:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:0.0:1775764857.453106:0:13894:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b600d4bbb0 00010000:00010000:0.0:1775764857.453110:0:13894:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:0.0:1775764857.453117:0:13894:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:0.0:1775764857.453128:0:13894:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:0.0:1775764857.453136:0:13894:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:0.0:1775764857.453319:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:0.0:1775764857.453337:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75c00 uarg=0000000000000000 00800000:00000004:0.0:1775764857.453344:0:13894:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 581, max_age 590 00000100:00100000:0.0:1775764857.453387:0:13894:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285248:192.168.202.104@tcp:41:lfs.0 00000100:00100000:0.0:1775764857.453980:0:13894:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c81a4cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764857.473466:0:13894:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285248:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764857.473539:0:13894:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:3.0:1775764857.473634:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.473652:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00800000:00000004:3.0:1775764857.473661:0:13894:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 582, max_age 590 00000100:00100000:3.0:1775764857.473700:0:13894:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285376:192.168.202.104@tcp:41:lfs.0 00000100:00100000:3.0:1775764857.473764:0:13894:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c2d1a6c0 going to sleep for 11 seconds 00010000:00080000:3.0:1775764857.477044:0:13894:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540000 x1862023861285376/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764873 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764857.477064:0:13894:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285376:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764857.477108:0:13894:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317437/353529 objects 1023743/1024000 00000080:00200080:3.0:1775764857.477175:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477194:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477202:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477205:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477210:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477213:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477216:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477219:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477223:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477226:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477230:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477233:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477237:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477239:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477243:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477246:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477250:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477252:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477256:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477260:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477263:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477266:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477270:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477274:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477287:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477291:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477294:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477298:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477303:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477306:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477310:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477312:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477316:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477318:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477322:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477325:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477328:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477331:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477335:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477337:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477341:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477344:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477348:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477351:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477355:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477359:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477362:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477365:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477369:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477378:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477383:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477386:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477390:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477392:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477395:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477398:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477402:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477404:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477407:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477410:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477414:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477416:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477420:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477423:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477425:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477429:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477432:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477435:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477439:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477441:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477444:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477448:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477452:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477455:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477466:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477470:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477474:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477477:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477480:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477526:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477532:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477537:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477540:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477544:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477548:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477551:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477555:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477559:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.477562:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.477565:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478437:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478452:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478464:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478468:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478472:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478475:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478478:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478482:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478516:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478534:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478539:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478543:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478547:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478550:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478554:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478558:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478561:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.478564:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.478567:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479071:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479080:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479084:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479087:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479090:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479094:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479097:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479100:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479104:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479108:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479111:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479115:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479118:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479121:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479124:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479137:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00800000:00000080:3.0:1775764857.479141:0:13894:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.479146:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.479152:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00020000:00000004:3.0:1775764857.479161:0:13894:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 582, max_age 590 00000100:00100000:3.0:1775764857.479221:0:13894:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285504:192.168.202.104@tcp:13:lfs.0 00000100:00100000:3.0:1775764857.479321:0:13894:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c2d1a6c0 going to sleep for 11 seconds 00010000:00080000:3.0:1775764857.483307:0:13894:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540000 x1862023861285504/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764873 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764857.483326:0:13894:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285504:192.168.202.104@tcp:13:lfs.0 00020000:00000004:3.0:1775764857.483352:0:13894:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:3.0:1775764857.483429:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.483446:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00020000:00000004:3.0:1775764857.483455:0:13894:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 582, max_age 590 00000100:00100000:3.0:1775764857.483517:0:13894:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285632:192.168.202.104@tcp:13:lfs.0 00000100:00100000:3.0:1775764857.484455:0:13894:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c2d1a6c0 going to sleep for 11 seconds 00010000:00080000:3.0:1775764857.494103:0:13894:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540000 x1862023861285632/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764873 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764857.494123:0:13894:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:13894:1862023861285632:192.168.202.104@tcp:13:lfs.0 00020000:00000004:3.0:1775764857.494149:0:13894:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:3.0:1775764857.494224:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494242:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494250:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494269:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494275:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494280:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494283:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494287:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494291:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494295:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494298:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494302:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494305:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494308:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494312:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494315:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494318:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494321:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494325:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494327:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494331:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494333:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494337:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494340:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494344:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494347:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494350:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494353:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494364:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494369:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494373:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494376:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494380:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494383:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494387:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494390:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494394:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494397:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494401:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494405:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494408:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494412:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494415:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494417:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494421:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494424:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494428:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494431:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494434:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494438:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494443:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494447:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494451:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494461:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494466:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494470:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494474:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494477:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494480:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494522:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494528:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494532:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494536:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494539:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494543:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494547:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494551:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494554:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494559:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494563:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494568:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494571:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494574:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494577:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494592:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494598:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494603:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494606:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494618:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494622:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494626:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494629:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494633:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494636:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494640:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494643:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494646:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494649:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494653:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494656:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494659:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494662:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494666:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494669:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494672:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494675:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494679:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494682:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494686:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494689:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494693:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494697:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494700:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494710:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494716:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494720:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494723:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494727:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494731:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494734:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494737:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494741:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494744:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494748:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494751:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494755:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494759:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494762:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494765:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494768:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494772:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494775:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494779:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494782:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200080:3.0:1775764857.494786:0:13894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffeb9600380 00020000:00000080:3.0:1775764857.494789:0:13894:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c4400 uarg=0000000000000000 00000080:00200008:3.0:1775764857.494814:0:13894:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:1.0:1775764858.288919:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764859.311172:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764860.331618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764860.783840:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861285760:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764860.783950:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861286016:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764860.783987:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861286272:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764860.784013:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764860.784018:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861285888:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764860.784040:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861286144:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764860.792370:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861285888:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764860.792392:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8def100 x1862023861286144/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764876 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764860.792418:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861286144:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764860.792433:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8deca80 x1862023861286016/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764876 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764860.792441:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861286016:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764860.792451:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9c8dedf80 x1862023861286272/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764876 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764860.792459:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861286272:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764860.795378:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861285760:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764861.361682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764862.379726:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764863.404013:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764864.432990:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764865.454761:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764865.620271:0:14497:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == insanity test 1: MDS/MDS failure ====================== 16:01:03 (1775764863) 00000100:00100000:1.0:1775764865.900000:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861286400:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764865.900183:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286656:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764865.900214:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286912:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764865.900247:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1775764865.900253:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286528:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764865.900276:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286784:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764865.906408:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286528:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764865.906430:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2401:1862023861286400:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764865.906446:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09df100 x1862023861286656/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764881 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764865.906459:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286656:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764865.906470:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09ded80 x1862023861286912/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764881 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764865.906478:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286912:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764865.913279:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09dfb80 x1862023861286784/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764881 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764865.913304:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861286784:192.168.202.104@tcp:400:kworker.0 00010000:00010000:3.0:1775764866.475638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.501095:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764867.811918:0:14535:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764867.811934:0:14535:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764867.811966:0:14535:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764867.811991:0:14535:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764867.812010:0:14535:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1775764867.812020:0:14535:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764867.812033:0:14535:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764867.812044:0:14535:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764867.812056:0:14535:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6018afcb8 00010000:00010000:2.0:1775764867.812060:0:14535:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764867.812070:0:14535:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200008:2.0:1775764867.812700:0:14535:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764867.812704:0:14535:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764867.812712:0:14535:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764867.812720:0:14535:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1775764867.812727:0:14535:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1775764867.812732:0:14535:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764867.812735:0:14535:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764867.812737:0:14535:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764867.812740:0:14535:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6018afcb8 00010000:00010000:2.0:1775764867.812742:0:14535:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1775764867.812746:0:14535:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1775764867.812835:0:14535:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764867.812844:0:14535:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764867.812849:0:14535:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1775764867.812855:0:14535:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1775764867.812860:0:14535:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764867.812863:0:14535:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764867.812864:0:14535:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764867.812867:0:14535:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6018afbb0 00010000:00010000:2.0:1775764867.812869:0:14535:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1775764867.812873:0:14535:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1775764867.812878:0:14535:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764867.812883:0:14535:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:1.0:1775764867.813674:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:1.0:1775764867.813697:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c2d8dc00 uarg=0000000000000000 00800000:00000004:1.0:1775764867.813707:0:14535:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 591, max_age 600 00000100:00100000:1.0:1775764867.813768:0:14535:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287040:192.168.202.104@tcp:41:lfs.0 00000100:00100000:1.0:1775764867.814012:0:14535:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c8c57300 going to sleep for 11 seconds 00000100:00100000:3.0:1775764867.815959:0:14535:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287040:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764867.816203:0:14535:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: update ffffa0e9d095d190 cache blocks 317406/353529 objects 1023722/1024000 00000080:00200080:3.0:1775764867.816312:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.816333:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00800000:00000004:3.0:1775764867.816341:0:14535:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: age 591, max_age 600 00000100:00100000:3.0:1775764867.816391:0:14535:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287168:192.168.202.104@tcp:41:lfs.0 00000100:00100000:3.0:1775764867.816479:0:14535:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c2d1a900 going to sleep for 11 seconds 00010000:00080000:3.0:1775764867.819890:0:14535:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540000 x1862023861287168/t0(0) o41->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 1775764883 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764867.819926:0:14535:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287168:192.168.202.104@tcp:41:lfs.0 00800000:00000004:3.0:1775764867.819968:0:14535:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0001-mdc-ffffa0e9c80cb000: update ffffa0e9d04c37d0 cache blocks 317437/353529 objects 1023743/1024000 00000080:00200080:3.0:1775764867.820071:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820095:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820104:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820107:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820110:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820112:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820115:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820117:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820119:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820120:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820122:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820140:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820143:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820146:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820148:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820151:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820153:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820154:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820156:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820158:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820160:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820161:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820163:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820165:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820167:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820168:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820170:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820172:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820174:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820175:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820177:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820179:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820181:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820182:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820184:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820186:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820195:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820197:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820199:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820201:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820203:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820205:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820207:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820208:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820210:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820212:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820214:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820215:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820217:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820219:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820221:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820222:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820224:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820226:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820228:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820229:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820231:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820233:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820234:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820236:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820238:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820244:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820247:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820249:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820252:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820253:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820255:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820257:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820259:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820260:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820262:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820264:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820266:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820268:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820269:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820271:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820273:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820274:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820276:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820278:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820280:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820281:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820283:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820285:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820287:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820288:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820295:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820297:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820300:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820302:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820304:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820305:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820307:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820309:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820311:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820312:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820314:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820316:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820318:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820319:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820321:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820323:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820324:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820326:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820328:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820329:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820331:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820333:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820335:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820336:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820338:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820343:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820346:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820348:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820350:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820351:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820353:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820355:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820357:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820358:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820360:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820362:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820364:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820365:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820367:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00800000:00000080:3.0:1775764867.820369:0:14535:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00000080:00200080:3.0:1775764867.820371:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:3.0:1775764867.820376:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9d01c5c00 uarg=0000000000000000 00020000:00000004:3.0:1775764867.820383:0:14535:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: age 591, max_age 600 00000100:00100000:3.0:1775764867.820434:0:14535:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287296:192.168.202.104@tcp:13:lfs.0 00000100:00100000:3.0:1775764867.820560:0:14535:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9c2d1a900 going to sleep for 11 seconds 00010000:00080000:0.0:1775764867.838257:0:14535:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d0540000 x1862023861287296/t0(0) o13->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764883 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764867.838311:0:14535:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287296:192.168.202.104@tcp:13:lfs.0 00020000:00000004:0.0:1775764867.838344:0:14535:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffffa0e9c80cb000: update ffffa0e9d076b7d0 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:0.0:1775764867.838437:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.838458:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00020000:00000004:0.0:1775764867.838468:0:14535:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: age 591, max_age 600 00000100:00100000:0.0:1775764867.838538:0:14535:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287424:192.168.202.104@tcp:13:lfs.0 00000100:00100000:0.0:1775764867.838633:0:14535:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d07d7d80 going to sleep for 11 seconds 00010000:00080000:0.0:1775764867.848964:0:14535:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa0e9d09ded80 x1862023861287424/t0(0) o13->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:7/4 lens 224/368 e 0 to 0 dl 1775764883 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764867.848990:0:14535:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job lfs:6a4854d3-5a79-4a54-82b7-484d046ae78a:14535:1862023861287424:192.168.202.104@tcp:13:lfs.0 00020000:00000004:0.0:1775764867.849041:0:14535:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffffa0e9c80cb000: update ffffa0e9c8d20450 cache blocks 901365/958279 objects 261712/262144 00000080:00200080:0.0:1775764867.849130:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849152:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849161:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849165:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849170:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849173:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849178:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849181:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849185:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849188:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849192:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849195:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849222:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849226:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849230:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849232:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849236:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849238:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849242:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849245:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849248:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849251:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849255:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849258:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849261:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849265:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849269:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849272:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849277:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849280:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849284:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849287:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849291:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849294:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849298:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849301:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849305:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849314:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849318:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849322:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849326:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849329:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849333:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849336:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849339:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849342:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849345:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849349:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849352:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849355:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849358:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849362:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849365:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849369:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849372:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849375:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849378:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849381:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849385:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849388:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849391:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849394:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849405:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849409:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849413:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849417:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849421:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849425:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849429:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849432:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.849436:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.849439:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850271:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850282:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850286:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850290:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850293:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850297:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850301:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850304:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850308:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850311:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850315:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850318:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850322:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850325:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850328:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850344:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850349:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850352:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850356:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850359:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850362:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850366:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850370:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850374:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850377:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850381:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850384:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850388:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850391:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850394:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850398:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850401:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850405:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850408:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850412:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850415:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850418:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850422:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850425:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850429:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850456:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850461:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850466:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850470:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850474:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850477:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850481:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850501:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850507:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850511:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850515:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850519:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200080:0.0:1775764867.850523:0:14535:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc1c81c4c0 00020000:00000080:0.0:1775764867.850526:0:14535:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8d75400 uarg=0000000000000000 00000080:00200008:0.0:1775764867.850545:0:14535:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00010000:00010000:0.0:1775764868.528172:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764869.547596:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764870.571772:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764871.024283:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861287552:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764871.024390:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861287808:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764871.024423:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861288064:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764871.024469:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764871.024476:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861287680:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764871.024533:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861287936:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764871.068191:0:2403:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-MDT0000-mdc-ffffa0e9c80cb000 of lustre-MDT0000_UUID@192.168.202.104@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1775764871.068203:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764871.068208:0:2403:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa0e9c80cb000: Connection to lustre-MDT0000 (at 192.168.202.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764871.083378:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764871.083393:0:2403:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 438 00000100:00080000:2.0:1775764871.083409:0:2403:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764871.083438:0:2403:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764871.083470:0:2403:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861288192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764871.083538:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861287680:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764871.083575:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0712680 x1862023861287936/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764887 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764871.083586:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861287936:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764871.083599:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0711500 x1862023861287808/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764887 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764871.083607:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861287808:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764871.083619:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0713800 x1862023861288064/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764887 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764871.083627:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861288064:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764871.083692:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861288192:192.168.202.104@tcp:38:ptlrpcd_01_00.0 00000100:00080000:3.0:1775764871.133635:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764871.133650:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764871.133656:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861288192:192.168.202.104@tcp:38:ptlrpcd_01_00.0 00010000:00010000:0.0:1775764871.596072:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764872.621314:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764873.644520:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764874.671758:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764875.692397:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764876.139810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764876.139835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764876.139843:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764876.139861:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 604 00000100:00080000:2.0:1775764876.139878:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1775764876.139886:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764876.139926:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764876.139949:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861288448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764876.140052:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861288448:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764876.140092:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861288576:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764876.140203:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861288320:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764876.141468:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861288832:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764876.143887:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861288704:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764876.145180:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764876.145198:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764876.145204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861288448:192.168.202.104@tcp:38:kworker.0 00010000:00080000:2.0:1775764876.156561:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0710e00 x1862023861288832/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764892 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764876.156587:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861288832:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764876.156883:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0713480 x1862023861288704/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764892 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764876.156896:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861288704:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764876.160509:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0712680 x1862023861288576/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764892 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764876.160537:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861288576:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764876.715713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764877.739823:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764878.763732:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764879.791123:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764880.812513:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764881.260587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764881.260595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764881.260600:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764881.260615:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 609 00000100:00080000:0.0:1775764881.260632:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1775764881.260665:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764881.260681:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764881.260704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861289088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764881.264164:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861289088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775764881.264514:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861288960:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764881.264558:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861289344:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764881.264590:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764881.264598:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861289216:192.168.202.104@tcp:400:kworker.0 00000100:00100000:1.0:1775764881.264651:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861289472:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764881.271258:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d09df480 x1862023861289216/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764881.271284:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861289216:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764881.271297:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d09dc000 x1862023861289472/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764881.271307:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861289472:192.168.202.104@tcp:400:kworker.0 00010000:00080000:0.0:1775764881.271319:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d09dce00 x1862023861289344/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764881.271328:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861289344:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764881.271441:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764881.271448:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764881.271452:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861289088:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775764881.835926:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764882.859870:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764883.887519:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764884.909337:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764885.935917:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764886.380030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764886.380036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764886.380041:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764886.380051:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 614 00000100:00080000:3.0:1775764886.380062:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764886.380069:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764886.380079:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764886.380097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861289728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764886.380160:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861289856:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764886.380312:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861289600:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764886.380385:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861290112:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764886.383096:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861289728:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764886.384483:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861289984:192.168.202.104@tcp:400:kworker.0 00010000:00080000:2.0:1775764886.393039:0:2404:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0543b80 x1862023861289984/t0(0) o400->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764902 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764886.393060:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861289984:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764886.393273:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0543800 x1862023861290112/t0(0) o400->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764902 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764886.393285:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861290112:192.168.202.104@tcp:400:kworker.0 00010000:00080000:3.0:1775764886.401200:0:2403:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d0541c00 x1862023861289856/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764902 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764886.401221:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861289856:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764886.401879:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764886.401889:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:0.0:1775764886.401894:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861289728:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764886.956479:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1775764887.405533:0:2403:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764871/real 1775764871] req@ffffa0e9d0712a00 x1862023861287552/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764887 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764887.436623:0:2403:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa0e9d0712a00 x1862023861287552/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764887 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764887.436641:0:2403:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1775764887.436646:0:2403:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.202.104@tcp: Connection to MGS (at 192.168.202.104@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1775764887.452046:0:2403:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.202.104@tcp for MGC192.168.202.104@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1775764887.452051:0:2403:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1775764887.452060:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9d0712a00 x1862023861287552/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764887 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764887.452075:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9d0713800 x1862023861288320/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764892 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764887.452104:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9d09ded80 x1862023861288960/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764887.452124:0:2403:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0e9d0540380 x1862023861289600/t0(0) o400->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764902 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1775764887.452133:0:2403:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1775764887.452136:0:2403:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1775764887.452138:0:2403:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1775764887.452141:0:2403:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:3.0:1775764887.452189:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2bf32c54-7e4b-4438-8419-d956344e0e00:2403:1862023861287552:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764887.452705:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2402:1862023861288960:192.168.202.104@tcp:400:kworker.0 00000100:00080000:1.0:1775764887.453575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764887.453584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764887.453587:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764887.453594:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 436 00000100:00080000:1.0:1775764887.453611:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775764887.453661:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861290240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764887.453692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764887.453696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764887.453697:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764887.453701:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 619 00000100:00080000:1.0:1775764887.453704:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764887.453708:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775764887.453726:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764887.453738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861290368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764887.453774:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:1.0:1775764887.453778:0:36:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffffa0e9c80cb000: disconnect after 20s idle 00000100:00080000:1.0:1775764887.456332:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:1.0:1775764887.456338:0:36:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffffa0e9c80cb000: disconnect after 20s idle 00000100:00100000:1.0:1775764887.456373:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764887.456408:0:2401:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861290496:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764887.457562:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861290240:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775764887.457666:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290368:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775764887.457695:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290624:192.168.202.104@tcp:9:kworker.0 00000100:00100000:0.0:1775764887.457731:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290752:192.168.202.104@tcp:9:kworker.0 00000100:00100000:2.0:1775764887.458627:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861289600:192.168.202.104@tcp:400:kworker.0 00000100:00100000:2.0:1775764887.458664:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2bf32c54-7e4b-4438-8419-d956344e0e00:2404:1862023861288320:192.168.202.104@tcp:400:kworker.0 00010000:00080000:1.0:1775764887.463695:0:2401:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9c8ded500 x1862023861290496/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764903 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764887.463723:0:2401:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2401:1862023861290496:192.168.202.104@tcp:400:kworker.0 00000100:00080000:0.0:1775764887.464286:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764887.464293:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:0.0:1775764887.464297:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290368:192.168.202.104@tcp:38:kworker.0 00010000:00080000:0.0:1775764887.464311:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9c8deea00 x1862023861290624/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764887.464323:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9c8deea00 x1862023861290624/t0(0) o9->lustre-OST0000-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764887.464330:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d071e000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1775764887.464333:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290624:192.168.202.104@tcp:9:kworker.0 00010000:00080000:0.0:1775764887.464345:0:2400:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9c8ded880 x1862023861290752/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764887.464353:0:2400:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa0e9c8ded880 x1862023861290752/t0(0) o9->lustre-OST0001-osc-ffffa0e9c80cb000@192.168.202.104@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764897 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764887.464360:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d07fd800 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1775764887.464363:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290752:192.168.202.104@tcp:9:kworker.0 00010000:00010000:3.0:1775764887.979675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764889.008284:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764890.030553:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764891.051658:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764891.501373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764891.501382:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764891.501401:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764891.501405:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764891.501410:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764891.501419:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 621 00000100:00080000:0.0:1775764891.501430:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764891.501437:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764891.501450:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764891.501479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861290880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764891.501594:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764891.501610:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861291008:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764891.501951:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290880:192.168.202.104@tcp:38:kworker.0 00010000:00080000:0.0:1775764891.506322:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d09dce00 x1862023861291008/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764907 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764891.506357:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861291008:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764891.506442:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764891.506449:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764891.506454:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861290880:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775764892.076553:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764892.524147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764892.524158:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764892.524161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764892.524165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764892.524170:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764892.524182:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 625 00000100:00080000:0.0:1775764892.524195:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764892.524203:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764892.524236:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764892.524269:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861291136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764892.524421:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861291264:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764892.526783:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861291136:192.168.202.104@tcp:38:kworker.0 00010000:00080000:1.0:1775764892.527273:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9d09dc000 x1862023861291264/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764908 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764892.527289:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861291264:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764892.529393:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764892.529406:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764892.529412:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861291136:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764893.104086:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764894.123680:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764895.147615:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764895.381453:0:14656:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764895.381472:0:14656:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764895.381623:0:14656:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764895.381719:0:14656:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1775764895.381739:0:14656:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1775764895.381746:0:14656:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764895.381755:0:14656:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764895.381757:0:14656:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764895.381765:0:14656:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014a3cb8 00010000:00010000:2.0:1775764895.381768:0:14656:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1775764895.381775:0:14656:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200008:2.0:1775764895.382343:0:14656:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764895.382348:0:14656:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764895.382355:0:14656:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764895.382362:0:14656:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1775764895.382372:0:14656:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1775764895.382377:0:14656:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764895.382381:0:14656:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764895.382382:0:14656:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764895.382385:0:14656:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014a3cb8 00010000:00010000:2.0:1775764895.382387:0:14656:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1775764895.382392:0:14656:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:2.0:1775764895.382876:0:14656:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88),name=/ 00000002:00010000:2.0:1775764895.382889:0:14656:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764895.382897:0:14656:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:2.0:1775764895.382907:0:14656:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:2.0:1775764895.382914:0:14656:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764895.382919:0:14656:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) 00000080:00010000:2.0:1775764895.382921:0:14656:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa0e9d110ba88) for lock 0x64300da17553cc26 00000080:00010000:2.0:1775764895.382925:0:14656:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x64300da17553cc26 from it ffffb6b6014a3bb0 00010000:00010000:2.0:1775764895.382928:0:14656:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:2.0:1775764895.382940:0:14656:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa0e9c80cb000 lock: ffffa0e9c0031600/0x64300da17553cc26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x39b1144a2f76ee85 expref: -99 pid: 11135 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:2.0:1775764895.382949:0:14656:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764895.383043:0:14656:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa0e9d110ba88), flags 100000 00000080:00200080:2.0:1775764895.383222:0:14656:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa0e9d110ba88) cmd=c00866a4 arg=7ffc62af2490 00800000:00000080:2.0:1775764895.383240:0:14656:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffffa0e9c80cb000: cmd=c00866a4 len=592 karg=ffffa0e9c8130800 uarg=0000000000000000 00800000:00000004:2.0:1775764895.383249:0:14656:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffffa0e9c80cb000: age 601, max_age 628 00000100:00080000:2.0:1775764895.383303:0:14656:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffffa0e9d0713800 x1862023861291392/t0(0) o41->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764895.383315:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764896.171852:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764896.427689:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764897.195708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764897.451837:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00100000:3.0:1775764897.643644:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764887/real 1775764887] req@ffffa0e9c8def480 x1862023861290240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764897 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764897.643682:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861290240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764897 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764897.643703:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764897.643720:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:3.0:1775764897.643724:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861290240:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775764897.643846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764897.643856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764897.643860:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764897.643870:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 621 00000100:00080000:1.0:1775764897.643884:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1775764897.643891:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775764897.643939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861291520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764897.643970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764897.643974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764897.643976:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764897.643980:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 626 00000100:00080000:1.0:1775764897.644004:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775764897.644010:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861291520:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775764897.644017:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775764897.644031:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764897.644050:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861291648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764897.644128:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764897.644134:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861291648:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775764897.644151:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861291776:192.168.202.104@tcp:400:kworker.0 00000100:00080000:3.0:1775764897.652328:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764897.652341:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764897.652346:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861291648:192.168.202.104@tcp:38:kworker.0 00010000:00080000:0.0:1775764897.652471:0:2402:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffffa0e9c8dedc00 x1862023861291776/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764913 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764897.652509:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861291776:192.168.202.104@tcp:400:kworker.0 00010000:00010000:2.0:1775764898.219693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764898.477867:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764899.244846:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764899.499750:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764900.267695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764900.523619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764901.291732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764901.547648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764902.315829:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764902.571612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775764902.763868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764902.763879:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764902.763882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764902.763885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764902.763890:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764902.763899:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 631 00000100:00080000:2.0:1775764902.763931:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764902.763937:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764902.763951:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764902.763988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861291904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764902.764048:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861291904:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764902.764059:0:2404:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861292032:192.168.202.104@tcp:400:kworker.0 00010000:00010000:3.0:1775764903.339636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764903.595643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764904.363668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764904.619591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764905.387667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764905.643647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764906.411642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764906.667632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764907.435670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764907.691598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775764907.883665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764907.883678:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764907.883680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764907.883683:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1775764907.883804:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861292160:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764908.459707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764908.715749:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764909.483624:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764909.739607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764910.507718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764910.763554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764911.531614:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764911.787573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764912.555628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764912.811574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775764913.003584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764913.003593:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764913.003596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775764913.003596:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764897/real 1775764897] req@ffffa0e9c8ded500 x1862023861291520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764912 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764913.003599:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1775764913.003630:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861291520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764912 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764913.003652:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775764913.003663:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1775764913.003672:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775764913.003675:0:2402:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861292288:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764913.003679:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861291520:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775764913.003710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764913.003716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775764913.003719:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764913.003725:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 631 00000100:00080000:2.0:1775764913.003732:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1775764913.003739:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775764913.003760:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861292416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764913.003806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764913.003811:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1775764913.003832:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861292416:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775764913.579649:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764913.835551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764914.603656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764914.859603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764915.627620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764915.883586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764916.651647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764916.907583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764917.675625:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764917.931577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775764918.123553:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764918.123560:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764918.123563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764918.123565:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:3.0:1775764918.123584:0:2404:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764902/real 1775764902] req@ffffa0e9d0710e00 x1862023861292032/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764918.123640:0:2403:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861292544:192.168.202.104@tcp:400:kworker.0 00000100:00100000:3.0:1775764918.130806:0:2404:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa0e9d0710e00 x1862023861292032/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764918.130820:0:2404:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1775764918.130825:0:2404:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffffa0e9c80cb000: Connection to lustre-MDT0001 (at 192.168.202.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1775764918.134272:0:2404:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00080000:2.0:1775764918.134326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764918.134330:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764918.134333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764918.134337:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764918.134339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764918.134343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775764918.134352:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764918.134360:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 481 00000100:00100000:3.0:1775764918.134364:0:2404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2404:1862023861292032:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764918.134383:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764918.134393:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764918.134412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861292672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764918.134452:0:2403:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-MDT0001-mdc-ffffa0e9c80cb000 of lustre-MDT0001_UUID@192.168.202.104@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1775764918.134456:0:2403:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffffa0e9d04cb000 already not connected (conn 7, was 6): CONNECTING 00000100:00100000:3.0:1775764918.134462:0:2403:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6a4854d3-5a79-4a54-82b7-484d046ae78a:2403:1862023861292544:192.168.202.104@tcp:400:kworker.0 00000100:00100000:0.0:1775764918.134478:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861292672:192.168.202.104@tcp:38:kworker.0 00000100:00080000:3.0:1775764918.135330:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764918.135334:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764918.135337:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861292672:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775764918.699611:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764918.955589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764919.723598:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764919.979575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764920.747651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764921.003594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764921.771584:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764922.027544:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764922.795673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764923.051597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00000400:1.0:1775764923.179587:0:2402:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764907/real 1775764907] req@ffffa0e9c8dec000 x1862023861292160/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764923 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764923.189276:0:2402:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa0e9c8dec000 x1862023861292160/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764923 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764923.189304:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861292160:192.168.202.104@tcp:400:kworker.0 00000100:00080000:2.0:1775764923.243569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764923.243579:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764923.243582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764923.243585:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764923.243587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764923.243591:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775764923.243595:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764923.243604:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 651 00000100:00080000:2.0:1775764923.243621:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1775764923.243627:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775764923.243638:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764923.243670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861292800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764923.243755:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861292800:192.168.202.104@tcp:38:kworker.0 00000100:00080000:3.0:1775764923.245157:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764923.245163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764923.245166:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861292800:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764923.819614:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764924.075554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764924.843564:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764925.099599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764925.867603:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764926.123567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764926.891645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764927.147586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00100000:3.0:1775764927.339593:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764902/real 1775764902] req@ffffa0e9d0712680 x1862023861291904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764927 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764927.339617:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861291904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764927 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764927.339632:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764927.339649:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:3.0:1775764927.339652:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861291904:192.168.202.104@tcp:38:kworker.0 00000100:00080000:0.0:1775764927.339691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764927.339698:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764927.339700:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764927.339704:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764927.339707:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764927.339712:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 636 00000100:00080000:0.0:1775764927.339720:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764927.339726:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764927.339736:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764927.339761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861292928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775764927.339781:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764927.339785:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775764927.339798:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764927.339802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 656 00000100:00080000:0.0:1775764927.339806:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1775764927.339810:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775764927.339815:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764927.339820:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861292928:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775764927.339827:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861293056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764927.339906:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293056:192.168.202.104@tcp:38:kworker.0 00000100:00080000:3.0:1775764927.341215:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764927.341221:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764927.341238:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293056:192.168.202.104@tcp:38:kworker.0 00000100:00080000:3.0:1775764927.341250:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764927.341253:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:3.0:1775764927.341257:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861292928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775764927.915628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764928.171554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764928.939695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764929.195583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00000400:1.0:1775764929.323599:0:2402:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764913/real 1775764913] req@ffffa0e9c8deed80 x1862023861292288/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764929.333194:0:2402:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa0e9c8deed80 x1862023861292288/t0(0) o400->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764929.333213:0:2402:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6a4854d3-5a79-4a54-82b7-484d046ae78a:2402:1862023861292288:192.168.202.104@tcp:400:kworker.0 00010000:00010000:1.0:1775764929.963628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764930.219552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764930.987615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764931.243558:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764932.011660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764932.267618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775764932.459589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764932.459599:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764932.459602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764932.459606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764932.459612:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764932.459622:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 660 00000100:00080000:3.0:1775764932.459633:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764932.459641:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764932.459653:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764932.459684:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861293184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764932.459702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764932.459705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764932.459706:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764932.459708:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 660 00000100:00080000:3.0:1775764932.459712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764932.459714:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775764932.459729:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293184:192.168.202.104@tcp:38:kworker.0 00000008:00100000:3.0:1775764932.459731:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764932.459741:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861293312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764932.459821:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293312:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764932.461132:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764932.461137:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764932.461140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293312:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764932.461147:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764932.461149:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764932.461151:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293184:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764933.035637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764933.291618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00100000:2.0:1775764933.483617:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764913/real 1775764913] req@ffffa0e9d0713b80 x1862023861292416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764933 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764933.483649:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861292416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764933 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764933.483687:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764933.483712:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775764933.483715:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861292416:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775764933.483777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764933.483786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764933.483791:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764933.483799:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 646 00000100:00080000:3.0:1775764933.483810:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764933.483816:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775764933.483851:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861293440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764933.483881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764933.483884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764933.483886:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764933.483892:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 666 00000100:00080000:3.0:1775764933.483896:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764933.483901:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775764933.483909:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861293440:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775764933.483911:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764933.483926:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861293568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764933.483934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764933.483936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764933.483938:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764933.483941:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 666 00000100:00080000:3.0:1775764933.483959:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764933.483963:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764933.483968:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764933.483977:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861293696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764933.484020:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293568:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764933.484047:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293696:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764933.485308:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764933.485316:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764933.485320:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293568:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764933.485329:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764933.485331:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764933.485333:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293696:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775764934.059618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764934.315580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764935.083680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764935.339583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764936.107646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764936.363567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764937.131577:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764937.387566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764938.155705:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764938.411569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775764938.603589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764938.603596:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764938.603599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764938.603602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764938.603605:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764938.603613:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 667 00000100:00080000:3.0:1775764938.603621:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764938.603626:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764938.603635:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764938.603662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861293824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764938.603681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764938.603684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764938.603685:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764938.603687:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 667 00000100:00080000:3.0:1775764938.603689:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764938.603703:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764938.603708:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764938.603709:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293824:192.168.202.104@tcp:38:kworker.0 00000100:00100000:3.0:1775764938.603717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861293952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764938.603796:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293952:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764938.605166:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764938.605172:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764938.605177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293952:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764938.605188:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764938.605191:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764938.605193:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861293824:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764939.179599:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764939.435582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764940.203609:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764940.459620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764941.227660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764941.483596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764942.251628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764942.507607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764943.275586:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764943.531565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775764943.723583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764943.723593:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764943.723596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764943.723599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764943.723604:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764943.723614:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 672 00000100:00080000:3.0:1775764943.723625:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764943.723641:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764943.723653:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764943.723684:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861294080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764943.723703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764943.723706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764943.723707:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764943.723710:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 672 00000100:00080000:3.0:1775764943.723713:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764943.723716:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775764943.723732:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294080:192.168.202.104@tcp:38:kworker.0 00000008:00100000:3.0:1775764943.723734:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764943.723746:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861294208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764943.723817:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294208:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764943.725033:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764943.725038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764943.725041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294208:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764943.725048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764943.725050:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764943.725052:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775764944.299659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764944.555581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764945.323647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764945.579563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764946.347672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764946.603604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764947.371654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764947.627543:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764948.395654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764948.651663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775764948.843572:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764948.843582:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764948.843587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764948.843590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764948.843595:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764948.843605:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 677 00000100:00080000:3.0:1775764948.843617:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764948.843623:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764948.843636:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764948.843674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861294336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764948.843701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764948.843704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764948.843705:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764948.843708:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 677 00000100:00080000:3.0:1775764948.843711:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764948.843713:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764948.843716:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764948.843727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861294464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764948.843746:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775764948.843858:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294464:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764948.845290:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764948.845296:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764948.845300:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294464:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764948.845307:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764948.845310:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764948.845313:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294336:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775764949.419613:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764949.675578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764950.443616:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764950.699576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764951.467645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764951.723590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764952.491661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764952.747585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764953.515709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764953.771632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775764953.963570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764953.963577:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764953.963579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764953.963582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764953.963585:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764953.963593:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 682 00000100:00080000:3.0:1775764953.963602:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764953.963607:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764953.963615:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764953.963641:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861294592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764953.963660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764953.963662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764953.963663:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764953.963666:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 682 00000100:00080000:3.0:1775764953.963668:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764953.963671:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764953.963674:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764953.963682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861294720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764953.963696:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294592:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764953.963805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294720:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764953.965155:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764953.965161:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-11) 00000100:00100000:2.0:1775764953.965165:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294720:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764953.965177:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764953.965180:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764953.965184:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294592:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764954.539681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764954.795612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764955.563622:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764955.819573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764956.587645:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764956.843621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764957.611640:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764957.867630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00100000:2.0:1775764958.059644:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764933/real 1775764933] req@ffffa0e9d0542d80 x1862023861293440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764958 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764958.059677:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861293440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764958 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764958.059696:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764958.059720:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775764958.059723:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861293440:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775764958.059793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764958.059802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764958.059807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764958.059816:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 667 00000100:00080000:3.0:1775764958.059826:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764958.059832:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775764958.059869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861294848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764958.059902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764958.059906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764958.059908:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764958.059911:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 687 00000100:00080000:3.0:1775764958.059915:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764958.059919:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764958.059928:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764958.059939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861294976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764958.059949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764958.059952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764958.059954:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764958.059958:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764958.059963:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764958.059988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861295104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764958.060122:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861294848:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775764958.060234:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294976:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764958.060258:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295104:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764958.061849:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764958.061857:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-19) 00000100:00100000:2.0:1775764958.061860:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861294976:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764958.061875:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764958.061879:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-11) 00000100:00100000:2.0:1775764958.061882:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295104:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775764958.635581:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764958.891579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764959.659762:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764959.915622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764960.683644:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764960.939627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764961.707668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764961.963644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764962.731743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764962.987586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775764963.179581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764963.179588:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764963.179590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764963.179593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764963.179597:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764963.179605:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 691 00000100:00080000:3.0:1775764963.179614:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764963.179620:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764963.179631:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764963.179659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861295232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764963.179680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764963.179682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764963.179683:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764963.179686:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775764963.179689:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764963.179698:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861295360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764963.179720:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775764963.179840:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295360:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775764963.755649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764964.011577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764964.779595:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764965.035547:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764965.803677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764966.059630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764966.827669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764967.083558:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764967.851598:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764968.107614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775764968.299746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764968.299753:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764968.299756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764968.299758:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764968.299760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764968.299761:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1775764968.875662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764969.131617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764969.899681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764970.155624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764970.923616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764971.179632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764971.947650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764972.203641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764972.971698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764973.227570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775764973.419623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764973.419636:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764973.419641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764973.419645:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764973.419648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764973.419652:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1775764973.995631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764974.251609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764975.019652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764975.275598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764976.043675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764976.299617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764977.067671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764977.323634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764978.091641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764978.347602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775764978.539634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764978.539646:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764978.539671:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764978.539675:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764978.539677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764978.539680:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1775764979.115656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764979.371636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764980.139591:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764980.395559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764981.163678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764981.419599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764982.187623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764982.443562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764983.211638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764983.467614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775764983.659619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764983.659629:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1775764983.659630:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764958/real 1775764958] req@ffffa0e9d0540700 x1862023861294848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764983.659632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764983.659639:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764983.659644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764983.659648:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1775764983.659668:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861294848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764983.659686:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764983.659697:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775764983.659700:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861294848:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775764983.659712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764983.659716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764983.659720:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764983.659727:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 691 00000100:00080000:1.0:1775764983.659739:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764983.659768:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775764983.659808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861295488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764983.659845:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764983.659848:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764983.659850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764983.659852:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1775764983.659859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861295488:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775764984.235618:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764984.491567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764985.259650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764985.515638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764986.283648:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764986.539588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764987.307647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764987.563618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764988.331652:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764988.587610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775764988.779571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764988.779578:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764988.779581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764988.779585:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764988.779587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764988.779589:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1775764988.779605:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764963/real 1775764963] req@ffffa0e9d0543800 x1862023861295360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764988 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764988.779636:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861295360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764988 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764988.779648:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764963/real 1775764963] req@ffffa0e9d0541c00 x1862023861295232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764988 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764988.779662:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861295232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764988 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764988.779684:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764988.779707:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775764988.779711:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295360:192.168.202.104@tcp:38:kworker.0 00000100:00080000:2.0:1775764988.779719:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764988.779727:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00080000:1.0:1775764988.779732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775764988.779732:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295232:192.168.202.104@tcp:38:kworker.0 00000100:00080000:1.0:1775764988.779746:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764988.779749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764988.779752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764988.779755:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764988.779763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 696 00000100:00080000:1.0:1775764988.779772:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764988.779806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775764988.779818:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764988.779849:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861295616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764988.779874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764988.779877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775764988.779879:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764988.779883:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 696 00000100:00080000:1.0:1775764988.779889:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764988.779893:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775764988.779898:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764988.779908:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295616:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775764988.779912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861295744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764988.780057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295744:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775764989.355619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764989.611611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764990.379679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764990.635625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764991.403645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764991.659606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764992.427627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764992.683635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764993.451675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764993.707633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775764993.899583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764993.899590:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764993.899592:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764993.899594:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764993.899596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764993.899597:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1775764994.475663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764994.731621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764995.499638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764995.755600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775764996.523699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764996.779630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775764997.547630:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764997.803617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775764998.571640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764998.827622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775764999.019654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764999.019662:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764999.019664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764999.019667:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764999.019668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764999.019670:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1775764999.595722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764999.851622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765000.619690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765000.875603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765001.643612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765001.899613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765002.667658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765002.923590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765003.691695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765003.947607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765004.139617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765004.139627:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775765004.139629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765004.139643:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775765004.139647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765004.139668:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1775765004.715612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765004.971603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000800:00000100:1.0:1775765005.227673:0:2392:0:(socklnd_cb.c:2376:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.202.104@tcp (192.168.202.104:988) timed out; the network or node may be down. 00010000:00010000:2.0:1775765005.739661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765005.995631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765006.763631:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765007.019574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765007.787653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765008.043609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00100000:2.0:1775765008.235682:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764983/real 1775764983] req@ffffa0e9c8deed80 x1862023861295488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765008.235706:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861295488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765008.235736:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765008.235754:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765008.235757:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861295488:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765008.235791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765008.235797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765008.235800:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765008.235806:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 717 00000100:00080000:0.0:1775765008.235815:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765008.235822:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765008.235852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861295872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765008.235874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765008.235878:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775765008.235881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765008.235885:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1775765008.235899:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861295872:192.168.202.104@tcp:250:kworker.0 00000400:00000100:0.0:1775765008.299701:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.104:988 00000400:00000100:0.0:1775765008.299739:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765008.299750:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765008.299803:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765008/real 1775765008] req@ffffa0e9d09de300 x1862023861295872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765008.299822:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861295872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765008.299830:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765008.299833:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765008.299836:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861295872:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775765008.811624:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765009.067605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765009.835636:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765010.091656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765010.859625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765011.115698:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765011.883745:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765012.139614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765012.907692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765013.163589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765013.355579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765013.355589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765013.355594:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765013.355604:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764988/real 1775764988] req@ffffa0e9c8dec380 x1862023861295744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765013.355606:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 741 00000100:00080000:0.0:1775765013.355621:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765013.355630:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765013.355639:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861295744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765013.355650:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764988/real 1775764988] req@ffffa0e9c8dec000 x1862023861295616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765013.355673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861296000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765013.355682:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861295616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765013.355712:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765013.355715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765013.355719:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775765013.355722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765013.355725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765013.355727:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765013.355731:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 722 00000100:00080000:0.0:1775765013.355734:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765013.355737:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00080000:0.0:1775765013.355742:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765013.355752:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295744:192.168.202.104@tcp:38:kworker.0 00000008:00100000:0.0:1775765013.355761:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765013.355763:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765013.355774:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765013.355782:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861296128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765013.355783:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861295616:192.168.202.104@tcp:38:kworker.0 00000100:00080000:0.0:1775765013.355812:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765013.355815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765013.355818:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765013.355825:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 722 00000100:00080000:0.0:1775765013.355829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765013.355834:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765013.355841:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765013.355851:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861296000:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765013.355858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861296256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765013.355952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765013.355988:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296256:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765013.931631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765014.187649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765014.955640:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765015.211588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765015.979692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765016.235662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:0.0:1775765016.427830:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.104:988 00000400:00000100:0.0:1775765016.427874:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765016.427903:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:0.0:1775765016.427949:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:0.0:1775765016.427963:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765016.428015:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765013/real 1775765016] req@ffffa0e9d09df480 x1862023861296256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765016.428045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861296256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765016.428059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765016.428065:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765016.428079:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765016.428089:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765013/real 1775765016] req@ffffa0e9d09dea00 x1862023861296000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765016.428105:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861296000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765016.428114:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765016.428118:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765016.428122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861296000:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765016.428130:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765013/real 1775765016] req@ffffa0e9d09de300 x1862023861296128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765016.428145:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861296128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765016.428153:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765016.428157:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765016.428162:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296128:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765017.003654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765017.259572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765018.027682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765018.283637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765018.475747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765018.475758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765018.475763:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765018.475774:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 746 00000100:00080000:0.0:1775765018.475785:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765018.475792:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765018.475836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861296384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765018.475868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765018.475872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765018.475874:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765018.475878:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 746 00000100:00080000:0.0:1775765018.475882:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765018.475887:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765018.475899:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765018.475916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861296512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765018.475941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765018.475945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765018.475947:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765018.475952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861296384:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765018.475954:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 746 00000100:00080000:0.0:1775765018.475987:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765018.475992:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765018.475999:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765018.476013:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861296640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765018.476161:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296512:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765018.476204:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296640:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765019.051670:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765019.307583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:1.0:1775765019.499897:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.104:988 00000400:00000100:1.0:1775765019.499933:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765019.499950:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775765019.499988:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775765019.500009:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765019.500032:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765018/real 1775765019] req@ffffa0e9d09dce00 x1862023861296512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765019.500072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861296512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765019.500086:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765019.500092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765019.500097:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296512:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765019.500107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765018/real 1775765019] req@ffffa0e9d09dc000 x1862023861296640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765019.500120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861296640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765019.500128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765019.500132:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765019.500137:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296640:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765019.500145:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765018/real 1775765019] req@ffffa0e9d09dc380 x1862023861296384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765019.500160:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861296384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765019.500169:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765019.500173:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765019.500204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861296384:192.168.202.104@tcp:250:kworker.0 00010000:00010000:3.0:1775765020.075623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765020.331594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765021.099685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765021.355611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765022.123704:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765022.379619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765023.147670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765023.403587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765023.595687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765023.595706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765023.595711:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765023.595719:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 752 00000100:00080000:1.0:1775765023.595728:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765023.595734:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765023.595767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861296768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765023.595800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765023.595814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765023.595815:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765023.595818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 752 00000100:00080000:1.0:1775765023.595820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765023.595837:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765023.595844:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765023.595854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861296896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765023.595862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765023.595865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765023.595868:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765023.595869:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861296768:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765023.595872:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 752 00000100:00080000:1.0:1775765023.595875:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765023.595884:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765023.595887:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765023.595897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861297024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765023.595975:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765023.595996:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297024:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765024.171687:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765024.427628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765025.195718:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765025.451602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765026.219584:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765026.475596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:0.0:1775765026.667744:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.104:988 00000400:00000100:0.0:1775765026.667783:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765026.667799:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:0.0:1775765026.667836:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:0.0:1775765026.667843:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765026.667891:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765023/real 1775765026] req@ffffa0e9c8ded180 x1862023861296896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765026.667922:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861296896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765026.667935:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765026.667941:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765026.667945:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861296896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765026.667953:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765023/real 1775765026] req@ffffa0e9c8dec000 x1862023861297024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765026.667965:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861297024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765026.667988:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765026.667992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765026.667996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765026.668003:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765023/real 1775765026] req@ffffa0e9c8dee300 x1862023861296768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765026.668018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861296768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765026.668025:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765026.668028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765026.668031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861296768:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775765027.243634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765027.499595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765028.267658:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765028.523593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765028.715603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765028.715613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765028.715618:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765028.715629:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 757 00000100:00080000:1.0:1775765028.715641:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765028.715648:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765028.715690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861297152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765028.715718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765028.715740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765028.715742:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765028.715748:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 757 00000100:00080000:1.0:1775765028.715754:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765028.715759:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765028.715776:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765028.715794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861297280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765028.715794:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861297152:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765028.715827:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765028.715832:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765028.715834:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765028.715839:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 757 00000100:00080000:1.0:1775765028.715844:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765028.715849:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765028.715855:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765028.715870:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861297408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765028.715897:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765028.715975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765028/real 1775765028] req@ffffa0e9c8dec380 x1862023861297152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765028.716000:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861297152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765028.716014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765028.716019:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765028.716023:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861297152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765028.716075:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765028.716104:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765028.716118:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765028/real 1775765028] req@ffffa0e9c8deed80 x1862023861297280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765028.716133:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861297280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765028.716142:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765028.716146:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765028.716151:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297280:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765028.716158:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765028.716182:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765028.716202:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765028/real 1775765028] req@ffffa0e9c8ded500 x1862023861297408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765028.716215:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861297408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765028.716223:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765028.716227:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765028.716230:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297408:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765029.291590:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765029.547624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765030.315610:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765030.571568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765031.339653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765031.595579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765032.363585:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765032.619576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765033.387672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765033.643559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765033.835641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765033.835652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765033.835657:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765033.835669:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 762 00000100:00080000:1.0:1775765033.835681:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765033.835687:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765033.835779:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861297536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765033.835814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765033.835818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765033.835820:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765033.835825:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 762 00000100:00080000:1.0:1775765033.835829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765033.835835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765033.835848:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765033.835864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861297664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765033.835869:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861297536:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765033.835887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765033.835891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765033.835893:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765033.835898:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 762 00000100:00080000:1.0:1775765033.835903:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765033.835907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765033.835913:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765033.835925:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861297792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765033.835954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765033.836007:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765033/real 1775765033] req@ffffa0e9c8dedc00 x1862023861297536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765033.836032:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861297536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765033.836042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765033.836045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765033.836048:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861297536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765033.836071:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765033.836088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765033.836096:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765033/real 1775765033] req@ffffa0e9c8def800 x1862023861297664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765033.836104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861297664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765033.836110:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765033.836112:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765033.836115:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765033.836118:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765033.836141:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765033.836148:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765033/real 1775765033] req@ffffa0e9c8def480 x1862023861297792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765033.836156:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861297792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765033.836161:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765033.836163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765033.836165:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861297792:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765034.411672:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765034.667618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765035.435622:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765035.691587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765036.459654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765036.715581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765037.483604:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765037.739604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765038.507593:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765038.763571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765038.955614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765038.955622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765038.955626:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765038.955634:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 767 00000100:00080000:2.0:1775765038.955643:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765038.955649:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765038.955682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861297920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765038.955707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765038.955710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765038.955726:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765038.955732:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 767 00000100:00080000:2.0:1775765038.955736:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765038.955740:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765038.955750:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765038.955762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861298048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765038.955764:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861297920:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765038.955789:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765038.955791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765038.955793:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765038.955796:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 767 00000100:00080000:2.0:1775765038.955799:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765038.955802:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765038.955806:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765038.955818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861298176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765038.955924:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765038.955957:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298176:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765039.531635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765039.787587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765040.555698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765040.811597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765041.579594:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765041.835592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775765042.027669:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.104:988 00000400:00000100:2.0:1775765042.027703:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765042.027717:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765042.027754:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765042.027769:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765042.027793:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765038/real 1775765042] req@ffffa0e9d0712d80 x1862023861298048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765042.027818:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861298048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765042.027828:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765042.027832:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765042.027835:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765042.027843:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765038/real 1775765042] req@ffffa0e9d0711c00 x1862023861298176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765042.027873:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861298176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765042.027882:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765042.027887:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765042.027891:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765042.027899:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765038/real 1775765042] req@ffffa0e9d0713b80 x1862023861297920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765042.027912:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861297920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765042.027920:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765042.027924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765042.027927:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861297920:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775765042.603608:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765042.859554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765043.627653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765043.883638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765044.075625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765044.075635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765044.075641:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765044.075651:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 772 00000100:00080000:1.0:1775765044.075677:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765044.075683:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765044.075716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861298304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765044.075752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765044.075756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765044.075758:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765044.075763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 772 00000100:00080000:1.0:1775765044.075767:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765044.075771:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765044.075782:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765044.075798:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861298432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765044.075812:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765044.075815:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861298304:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765044.075816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765044.075819:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765044.075825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 772 00000100:00080000:1.0:1775765044.075829:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765044.075835:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765044.075840:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765044.075871:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861298560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765044.075906:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765044.075980:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765044/real 1775765044] req@ffffa0e9c8ded880 x1862023861298304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765044.076003:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861298304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765044.076016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765044.076020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765044.076025:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861298304:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765044.076067:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765044.076094:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765044.076109:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765044/real 1775765044] req@ffffa0e9c8deea00 x1862023861298432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765044.076120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861298432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765044.076128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765044.076140:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765044.076143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298432:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765044.076150:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765044.076172:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765044.076183:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765044/real 1775765044] req@ffffa0e9c8dee680 x1862023861298560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765044.076194:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861298560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765044.076201:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765044.076205:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765044.076208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298560:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765044.651604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765044.907575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765045.675701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765045.931597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765046.699695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765046.955581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765047.723581:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765047.979565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765048.747647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765049.003588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765049.195593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765049.195602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765049.195606:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765049.195614:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 777 00000100:00080000:1.0:1775765049.195623:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765049.195628:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765049.195661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861298688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765049.195685:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765049.195688:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765049.195689:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765049.195691:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 777 00000100:00080000:1.0:1775765049.195694:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765049.195696:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765049.195704:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765049.195712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861298816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765049.195720:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765049.195722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765049.195724:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765049.195726:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 777 00000100:00100000:0.0:1775765049.195732:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861298688:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765049.195743:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765049.195747:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765049.195751:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765049.195761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861298944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765049.195792:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765049.195826:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765049/real 1775765049] req@ffffa0e9c8dece00 x1862023861298688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765049.195861:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861298688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765049.195871:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765049.195875:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765049.195878:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861298688:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765049.195883:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765049.195901:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765049.195910:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765049/real 1775765049] req@ffffa0e9c8dedf80 x1862023861298816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765049.195918:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861298816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765049.195923:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765049.195925:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765049.195927:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298816:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765049.195945:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765049.195955:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765049.195961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765049/real 1775765049] req@ffffa0e9c8deca80 x1862023861298944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765049.195968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861298944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765049.195982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765049.195985:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765049.195987:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861298944:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765049.771725:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765050.027572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765050.795630:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765051.051600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765051.819648:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765052.075574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765052.843665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765053.099603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765053.867638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765054.123574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765054.315594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765054.315606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765054.315611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765054.315622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 782 00000100:00080000:2.0:1775765054.315635:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765054.315642:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765054.315686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861299072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765054.315717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765054.315720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765054.315721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765054.315724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 782 00000100:00080000:2.0:1775765054.315727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765054.315748:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765054.315758:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765054.315769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861299200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765054.315779:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765054.315782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765054.315783:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765054.315789:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 782 00000100:00100000:0.0:1775765054.315791:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861299072:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765054.315792:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765054.315798:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765054.315804:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765054.315811:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861299328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765054.315883:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765054.315932:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765054/real 1775765054] req@ffffa0e9d0710380 x1862023861299072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765054.315954:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861299072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765054.315966:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765054.315971:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765054.315976:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861299072:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765054.316004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765054.316029:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765054.316045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765054/real 1775765054] req@ffffa0e9d0711180 x1862023861299200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765054.316056:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861299200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765054.316064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765054.316068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765054.316072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765054.316085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765054.316107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765054.316119:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765054/real 1775765054] req@ffffa0e9d0710000 x1862023861299328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765054.316131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861299328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765054.316139:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765054.316143:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765054.316146:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299328:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765054.891715:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765055.147624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765055.915662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765056.171586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765056.939606:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765057.195628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765057.963631:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765058.219587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765058.987684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765059.243639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765059.435567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765059.435578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765059.435583:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765059.435594:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 787 00000100:00080000:2.0:1775765059.435606:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765059.435632:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765059.435671:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861299456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765059.435701:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765059.435705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765059.435707:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765059.435712:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 787 00000100:00080000:2.0:1775765059.435716:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765059.435721:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765059.435733:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765059.435754:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861299456:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765059.435755:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861299584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765059.435770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765059.435772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765059.435774:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765059.435777:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 787 00000100:00080000:2.0:1775765059.435779:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765059.435782:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765059.435786:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765059.435794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861299712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765059.435887:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299584:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765059.435922:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299712:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765060.011684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765060.267592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765061.035643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765061.291621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765062.059681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765062.315583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765062.507739:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.104:988 00000400:00000100:3.0:1775765062.507800:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765062.507820:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765062.507874:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765062.507891:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765062.507913:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765059/real 1775765062] req@ffffa0e9d0711180 x1862023861299584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765062.507946:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861299584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765062.507961:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765062.507979:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765062.507985:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299584:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765062.507994:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765059/real 1775765062] req@ffffa0e9d0710380 x1862023861299712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765062.508019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861299712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765062.508028:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765062.508032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765062.508036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299712:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765062.508045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765059/real 1775765062] req@ffffa0e9d0710000 x1862023861299456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765062.508059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861299456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765062.508068:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765062.508072:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765062.508076:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861299456:192.168.202.104@tcp:250:kworker.0 00010000:00010000:3.0:1775765063.083638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765063.339636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765064.107720:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765064.363628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765064.555607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765064.555618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765064.555623:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765064.555633:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 793 00000100:00080000:1.0:1775765064.555645:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765064.555652:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765064.555691:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861299840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765064.555716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765064.555720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765064.555722:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765064.555726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 793 00000100:00080000:1.0:1775765064.555730:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765064.555735:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765064.555747:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765064.555779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861299840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765064.555783:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861299968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765064.555798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765064.555802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765064.555804:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765064.555809:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 793 00000100:00080000:1.0:1775765064.555815:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765064.555819:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765064.555826:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765064.555840:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861300096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765064.555890:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765064.555940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765064/real 1775765064] req@ffffa0e9c8def100 x1862023861299840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765064.555972:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861299840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765064.555991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765064.555996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765064.556001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861299840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765064.556042:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765064.556071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765064.556099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765064/real 1775765064] req@ffffa0e9c8dec700 x1862023861299968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765064.556114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861299968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765064.556123:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765064.556127:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765064.556130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861299968:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765064.556137:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765064.556160:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765064.556173:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765064/real 1775765064] req@ffffa0e9c8defb80 x1862023861300096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765064.556185:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861300096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765064.556199:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765064.556202:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765064.556205:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300096:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765065.131709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765065.387597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765066.155615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765066.411576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765067.179613:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765067.435591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765068.203746:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765068.459594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765069.227570:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765069.483594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765069.675594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765069.675605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765069.675610:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765069.675621:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 798 00000100:00080000:1.0:1775765069.675632:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765069.675639:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765069.675679:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861300224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765069.675711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765069.675715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765069.675717:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765069.675736:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 798 00000100:00080000:1.0:1775765069.675742:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765069.675747:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765069.675753:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861300224:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775765069.675758:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765069.675777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861300352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765069.675789:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765069.675793:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765069.675795:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765069.675799:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 798 00000100:00080000:1.0:1775765069.675820:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765069.675833:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:1.0:1775765069.675837:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765069.675848:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765069.675862:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861300480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765069.675883:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765069/real 1775765069] req@ffffa0e9c8defb80 x1862023861300224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765069.675923:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861300224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765069.675934:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765069.675938:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765069.675943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861300224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765069.675967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765069.675994:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765069.676007:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765069/real 1775765069] req@ffffa0e9c8dec700 x1862023861300352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765069.676020:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861300352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765069.676042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765069.676044:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765069.676047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300352:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765069.676051:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765069.676074:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765069.676086:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765069/real 1775765069] req@ffffa0e9c8def100 x1862023861300480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765069.676098:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861300480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765069.676119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765069.676123:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765069.676126:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300480:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765070.251658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765070.507590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765071.275629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765071.531602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765072.299591:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765072.555646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765073.323622:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765073.579623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765074.347695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765074.603634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765074.795626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765074.795641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765074.795648:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765074.795663:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 803 00000100:00080000:1.0:1775765074.795680:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765074.795689:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765074.795750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861300608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765074.795815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765074.795820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765074.795823:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765074.795828:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 803 00000100:00080000:1.0:1775765074.795834:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765074.795840:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765074.795854:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765074.795873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861300736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765074.795888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765074.795893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765074.795896:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765074.795906:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861300608:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765074.795907:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 803 00000100:00080000:1.0:1775765074.795916:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765074.795927:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765074.795933:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765074.795963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861300864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765074.796044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765074.796095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765074/real 1775765074] req@ffffa0e9c8deca80 x1862023861300608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765074.796120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861300608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765074.796135:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765074.796141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765074.796145:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861300608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765074.796170:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765074.796194:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765074.796205:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765074/real 1775765074] req@ffffa0e9c8dedf80 x1862023861300736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765074.796214:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861300736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765074.796221:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765074.796224:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765074.796227:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300736:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765074.796244:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765074.796271:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765074.796304:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765074/real 1775765074] req@ffffa0e9c8dece00 x1862023861300864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765074.796317:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861300864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765074.796324:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765074.796327:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765074.796330:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861300864:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765075.371617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765075.627654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765076.395707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765076.651575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765077.419685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765077.675612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765078.443592:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765078.699586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765079.467675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765079.723563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765079.915575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765079.915583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765079.915588:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765079.915595:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 808 00000100:00080000:1.0:1775765079.915620:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765079.915627:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765079.915663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861300992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765079.915690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765079.915693:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765079.915695:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765079.915698:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 808 00000100:00080000:1.0:1775765079.915701:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765079.915705:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765079.915713:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765079.915727:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861301120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765079.915740:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861300992:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765079.915740:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765079.915748:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765079.915751:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765079.915757:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 808 00000100:00080000:1.0:1775765079.915761:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765079.915766:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765079.915772:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765079.915786:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861301248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765079.915810:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765079.915846:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765079/real 1775765079] req@ffffa0e9c8dee680 x1862023861300992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765079.915865:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861300992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765079.915875:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765079.915880:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765079.915883:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861300992:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765079.915906:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765079.915943:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765079.915959:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765079/real 1775765079] req@ffffa0e9c8deea00 x1862023861301120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765079.915981:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861301120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765079.915991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765079.915994:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765079.915996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301120:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765079.916003:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765079.916024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765079.916035:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765079/real 1775765079] req@ffffa0e9c8ded880 x1862023861301248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765079.916047:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861301248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765079.916055:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765079.916058:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765079.916061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301248:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765080.491648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765080.747609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765081.515643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765081.771546:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765082.539632:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765082.795594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765083.563645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765083.819660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765084.587583:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765084.843562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765085.035588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765085.035596:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765085.035599:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765085.035608:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 813 00000100:00080000:0.0:1775765085.035619:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765085.035624:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765085.035658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861301376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765085.035694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765085.035697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765085.035699:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765085.035701:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 813 00000100:00080000:0.0:1775765085.035704:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765085.035707:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765085.035715:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765085.035725:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861301504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765085.035732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765085.035735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765085.035736:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765085.035739:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 813 00000100:00080000:0.0:1775765085.035742:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765085.035746:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861301376:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765085.035747:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765085.035750:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765085.035761:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861301632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765085.035820:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765085.035878:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765085/real 1775765085] req@ffffa0e9d09ded80 x1862023861301376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765085.035908:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861301376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765085.035917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765085.035920:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765085.035924:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861301376:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765085.035930:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765085.035949:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765085.035958:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765085/real 1775765085] req@ffffa0e9d09df100 x1862023861301504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765085.035967:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861301504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765085.035973:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765085.035975:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765085.035977:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301504:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765085.036003:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765085.036019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765085.036026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765085/real 1775765085] req@ffffa0e9d09dfb80 x1862023861301632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765085.036045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861301632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765085.036051:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765085.036053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765085.036055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301632:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765085.611739:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765085.867620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765086.635641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765086.891548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765087.659616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765087.915583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765088.683665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765088.939577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765089.707632:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765089.963579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765090.155595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765090.155604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765090.155608:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765090.155615:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 818 00000100:00080000:2.0:1775765090.155625:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765090.155668:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765090.155706:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861301760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765090.155728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765090.155731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765090.155732:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765090.155735:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 818 00000100:00080000:2.0:1775765090.155748:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765090.155751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765090.155760:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765090.155765:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861301760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765090.155772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861301888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765090.155781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765090.155783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765090.155784:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765090.155788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 818 00000100:00080000:2.0:1775765090.155790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765090.155792:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765090.155795:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765090.155803:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861302016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765090.155860:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765090.155900:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765090/real 1775765090] req@ffffa0e9d0713b80 x1862023861301760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765090.155963:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861301760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765090.155975:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765090.155979:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765090.155984:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861301760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765090.156009:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765090.156036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765090.156050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765090/real 1775765090] req@ffffa0e9d0711c00 x1862023861301888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765090.156060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861301888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765090.156066:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765090.156068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765090.156070:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861301888:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765090.156075:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765090.156088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765090.156095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765090/real 1775765090] req@ffffa0e9d0712d80 x1862023861302016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765090.156102:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861302016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765090.156118:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765090.156120:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765090.156123:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302016:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765090.731580:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765090.987579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765091.755624:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765092.011610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765092.779635:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765093.035594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765093.803644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765094.059590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765094.827596:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765095.083591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765095.275591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765095.275602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765095.275619:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765095.275629:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 823 00000100:00080000:1.0:1775765095.275641:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765095.275648:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765095.275691:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861302144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765095.275731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765095.275736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765095.275738:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765095.275742:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 823 00000100:00080000:1.0:1775765095.275747:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765095.275753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765095.275767:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765095.275773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861302144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765095.275785:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861302272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765095.275821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765095.275834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765095.275839:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765095.275842:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 823 00000100:00080000:1.0:1775765095.275863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765095.275866:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765095.275871:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765095.275900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861302400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765095.275953:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765095.275997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302400:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765095.851696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765096.107626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765096.875667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765097.131645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765097.899656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765098.155601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775765098.348030:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775765098.348085:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765098.348101:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765098.348136:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765098.348146:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765098.348199:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765095/real 1775765098] req@ffffa0e9c8def800 x1862023861302272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765098.348236:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861302272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765098.348251:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765098.348258:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765098.348262:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765098.348272:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765095/real 1775765098] req@ffffa0e9c8dedc00 x1862023861302400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765098.348285:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861302400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765098.348292:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765098.348296:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765098.348313:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302400:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765098.348321:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765095/real 1775765098] req@ffffa0e9c8def480 x1862023861302144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765098.348335:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861302144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765098.348343:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765098.348347:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765098.348350:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861302144:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775765098.923663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765099.179562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765099.947580:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765100.203612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765100.395591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765100.395602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765100.395608:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765100.395618:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 828 00000100:00080000:0.0:1775765100.395631:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765100.395638:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765100.395684:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861302528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765100.395857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765100.395878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765100.395883:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765100.395895:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 828 00000100:00080000:0.0:1775765100.395909:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765100.395915:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861302528:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765100.395920:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765100.395938:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765100.395995:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861302656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765100.396012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765100.396017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765100.396019:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765100.396023:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 828 00000100:00080000:0.0:1775765100.396029:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765100.396036:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:2.0:1775765100.396038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:0.0:1775765100.396045:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765100.396062:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861302784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765100.396091:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765100/real 1775765100] req@ffffa0e9d09dc700 x1862023861302528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765100.396114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861302528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765100.396125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765100.396130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765100.396134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861302528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765100.396161:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765100.396192:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765100.396206:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765100/real 1775765100] req@ffffa0e9d09df800 x1862023861302656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765100.396219:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861302656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765100.396228:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765100.396232:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765100.396242:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302656:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765100.396249:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765100.396272:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765100.396284:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765100/real 1775765100] req@ffffa0e9d09dd880 x1862023861302784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765100.396296:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861302784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765100.396311:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765100.396315:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765100.396319:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861302784:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765100.971674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765101.227553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765101.995633:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765102.251554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765103.019643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765103.275581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765104.043751:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765104.299555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765105.067769:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765105.323640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765105.515630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765105.515641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765105.515646:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765105.515655:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 834 00000100:00080000:0.0:1775765105.515665:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765105.515672:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765105.515741:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861302912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765105.515790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765105.515800:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765105.515803:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765105.515811:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 834 00000100:00080000:0.0:1775765105.515818:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765105.515835:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861302912:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765105.515841:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765105.515855:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765105.515873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861303040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765105.515886:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765105.515890:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765105.515902:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765105.515905:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 834 00000100:00080000:0.0:1775765105.515910:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765105.515913:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765105.515919:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765105.515929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861303168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765105.515937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765105.516001:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765105/real 1775765105] req@ffffa0e9d09dd180 x1862023861302912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765105.516024:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861302912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765105.516058:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765105.516062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765105.516068:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861302912:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765105.516097:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765105.516143:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765105.516161:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765105/real 1775765105] req@ffffa0e9d09ddc00 x1862023861303040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765105.516173:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861303040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765105.516180:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765105.516183:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765105.516186:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765105.516192:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765105.516220:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765105.516228:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765105/real 1775765105] req@ffffa0e9d09dd500 x1862023861303168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765105.516238:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861303168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765105.516245:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765105.516247:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765105.516250:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303168:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765106.091569:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765106.347569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765107.115717:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765107.371570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765108.139610:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765108.395567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765109.163705:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765109.419626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765110.187615:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765110.443573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765110.635594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765110.635604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765110.635609:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765110.635620:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 839 00000100:00080000:1.0:1775765110.635631:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765110.635637:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765110.635694:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861303296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765110.635719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765110.635723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765110.635725:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765110.635729:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 839 00000100:00080000:1.0:1775765110.635733:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765110.635739:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765110.635750:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765110.635768:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861303296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765110.635781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861303424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765110.635790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765110.635792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765110.635794:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765110.635797:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 839 00000100:00080000:1.0:1775765110.635800:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765110.635803:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765110.635807:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765110.635817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861303552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765110.635839:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765110.635884:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765110/real 1775765110] req@ffffa0e9c8ded500 x1862023861303296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765110.635906:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861303296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765110.635917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765110.635922:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765110.635926:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861303296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765110.635951:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765110.635988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765110.636002:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765110/real 1775765110] req@ffffa0e9c8deed80 x1862023861303424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765110.636015:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861303424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765110.636024:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765110.636027:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765110.636039:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303424:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765110.636046:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765110.636070:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765110.636082:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765110/real 1775765110] req@ffffa0e9c8dec380 x1862023861303552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765110.636095:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861303552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765110.636103:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765110.636106:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765110.636110:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303552:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765111.211687:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765111.467615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765112.235588:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765112.491564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765113.259686:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765113.515587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765114.283646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765114.539678:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765115.307714:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765115.563620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765115.755625:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765115.755637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765115.755642:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765115.755655:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 844 00000100:00080000:2.0:1775765115.755669:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765115.755693:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765115.755736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861303680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765115.755783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765115.755786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765115.755788:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765115.755791:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 844 00000100:00080000:2.0:1775765115.755794:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765115.755797:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765115.755806:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765115.755816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861303808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765115.755824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765115.755827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765115.755828:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765115.755831:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 844 00000100:00080000:2.0:1775765115.755833:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765115.755835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765115.755839:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765115.755847:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861303680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765115.755850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861303936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765115.755942:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765115.755990:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765115/real 1775765115] req@ffffa0e9d0712680 x1862023861303680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765115.756008:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861303680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765115.756017:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765115.756020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765115.756024:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861303680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765115.756038:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765115.756057:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765115.756066:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765115/real 1775765115] req@ffffa0e9d0711f80 x1862023861303808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765115.756075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861303808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765115.756081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765115.756083:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765115.756085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303808:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765115.756109:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765115.756120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765115.756126:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765115/real 1775765115] req@ffffa0e9d0710700 x1862023861303936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765115.756134:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861303936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765115.756139:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765115.756141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765115.756143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861303936:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765116.331693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765116.587638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765117.355697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765117.611605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765118.379690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765118.635611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765119.403674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765119.659636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765120.427738:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765120.683646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765120.875635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765120.875648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765120.875654:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765120.875666:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 849 00000100:00080000:1.0:1775765120.875679:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765120.875687:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765120.875732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861304064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765120.875772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765120.875780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765120.875802:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765120.875810:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 849 00000100:00080000:1.0:1775765120.875834:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765120.875850:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765120.875859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861304064:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775765120.875884:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765120.875908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861304192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765120.875930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765120.875937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765120.875940:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765120.875948:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 849 00000100:00080000:1.0:1775765120.875954:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765120.875960:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775765120.875969:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775765120.875969:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765120.875988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861304320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765120.876026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765120/real 1775765120] req@ffffa0e9c8dee300 x1862023861304064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765120.876084:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861304064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765120.876099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765120.876104:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765120.876109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861304064:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765120.876139:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765120.876173:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765120.876190:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765120/real 1775765120] req@ffffa0e9c8dec000 x1862023861304192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765120.876205:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861304192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765120.876216:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765120.876220:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765120.876224:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765120.876231:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765120.876263:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765120.876283:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765120/real 1775765120] req@ffffa0e9c8ded180 x1862023861304320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765120.876307:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861304320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765120.876347:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765120.876355:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765120.876362:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304320:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765121.451623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765121.707647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765122.475713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765122.731707:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765123.499704:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765123.755620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765124.523636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765124.779598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765125.547725:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765125.803672:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765125.995684:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765125.995695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765125.995703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765125.995730:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 854 00000100:00080000:2.0:1775765125.995763:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765125.995774:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765125.995824:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861304448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765125.995871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765125.995874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765125.995876:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765125.995880:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 854 00000100:00080000:2.0:1775765125.995883:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765125.995886:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765125.995895:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765125.995909:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861304576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765125.995922:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861304448:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765125.995940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765125.995943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765125.995945:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765125.995951:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 854 00000100:00080000:2.0:1775765125.995955:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765125.995958:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765125.995964:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765125.995976:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861304704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765125.995997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765125.996074:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765125/real 1775765125] req@ffffa0e9d0710a80 x1862023861304448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765125.996107:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861304448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765125.996126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765125.996134:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765125.996141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861304448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765125.996188:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765125.996220:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765125.996237:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765125/real 1775765125] req@ffffa0e9d0710e00 x1862023861304576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765125.996250:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861304576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765125.996258:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765125.996261:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765125.996264:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304576:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765125.996277:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765125.996301:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765125.996315:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765125/real 1775765125] req@ffffa0e9d0712a00 x1862023861304704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765125.996331:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861304704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765125.996342:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765125.996347:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765125.996351:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304704:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765126.571782:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765126.827663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765127.595629:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765127.851582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765128.619672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765128.875588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765129.643634:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765129.899553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765130.667618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765130.923571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765131.115610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765131.115621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765131.115626:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765131.115636:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 859 00000100:00080000:0.0:1775765131.115648:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765131.115655:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765131.115698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861304832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765131.115729:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765131.115732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765131.115733:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765131.115736:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 859 00000100:00080000:0.0:1775765131.115741:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765131.115759:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765131.115769:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765131.115772:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861304832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765131.115782:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861304960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765131.115803:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765131.115806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765131.115809:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765131.115814:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 859 00000100:00080000:0.0:1775765131.115820:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765131.115825:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765131.115831:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765131.115845:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861305088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765131.115858:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765131.115905:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765131/real 1775765131] req@ffffa0e9d09de680 x1862023861304832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765131.115926:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861304832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765131.115938:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765131.115944:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765131.115960:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861304832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765131.115985:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765131.116014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765131.116027:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765131/real 1775765131] req@ffffa0e9d09dca80 x1862023861304960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765131.116036:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861304960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765131.116042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765131.116045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765131.116047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861304960:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765131.116052:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765131.116064:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765131.116070:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765131/real 1775765131] req@ffffa0e9d09ddf80 x1862023861305088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765131.116078:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861305088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765131.116082:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765131.116085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765131.116087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305088:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765131.691652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765131.947567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765132.715705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765132.971601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765133.739677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765133.995572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765134.763651:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765135.019652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765135.787717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765136.043640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765136.235691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765136.235713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765136.235721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765136.235736:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 864 00000100:00080000:2.0:1775765136.235755:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765136.235766:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765136.235841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861305216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765136.235908:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765136.235915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765136.235919:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765136.235926:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 864 00000100:00080000:2.0:1775765136.235935:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765136.235942:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765136.235961:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765136.235991:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861305344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765136.236004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861305216:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765136.236032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765136.236039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765136.236045:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765136.236066:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 864 00000100:00080000:2.0:1775765136.236074:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765136.236114:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765136.236120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765136.236133:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861305472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765136.236157:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765136.236206:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765136/real 1775765136] req@ffffa0e9d0713480 x1862023861305216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765136.236225:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861305216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765136.236235:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765136.236239:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765136.236243:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861305216:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765136.236278:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765136.236299:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765136.236309:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765136/real 1775765136] req@ffffa0e9d0711500 x1862023861305344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765136.236320:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861305344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765136.236326:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765136.236329:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765136.236332:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305344:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765136.236337:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765136.236367:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765136.236379:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765136/real 1775765136] req@ffffa0e9d0712300 x1862023861305472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765136.236393:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861305472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765136.236403:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765136.236407:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765136.236430:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305472:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765136.811677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765137.067636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765137.835719:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765138.091642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765138.859697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765139.115578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765139.883637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765140.139661:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765140.907637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765141.163645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765141.355619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765141.355634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765141.355640:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765141.355669:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 869 00000100:00080000:1.0:1775765141.355688:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765141.355695:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765141.355737:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861305600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765141.355765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765141.355768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765141.355770:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765141.355773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 869 00000100:00080000:1.0:1775765141.355777:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765141.355780:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765141.355789:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765141.355805:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861305728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765141.355811:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861305600:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765141.355835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765141.355839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765141.355841:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765141.355846:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 869 00000100:00080000:1.0:1775765141.355851:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765141.355855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765141.355861:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765141.355872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861305856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765141.355887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765141.355931:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765141/real 1775765141] req@ffffa0e9c8ded180 x1862023861305600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765141.355948:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861305600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765141.355956:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765141.355959:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765141.355962:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861305600:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765141.355992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765141.356012:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765141.356025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765141/real 1775765141] req@ffffa0e9c8dec000 x1862023861305728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765141.356039:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861305728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765141.356048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765141.356051:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765141.356055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305728:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765141.356063:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765141.356087:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765141.356099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765141/real 1775765141] req@ffffa0e9c8dee300 x1862023861305856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765141.356120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861305856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765141.356125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765141.356128:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765141.356130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861305856:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765141.931797:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765142.187624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765142.955606:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765143.211577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765143.979677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765144.235591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765145.003664:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765145.259609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765146.027668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765146.283637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765146.475615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765146.475626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765146.475632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765146.475642:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 874 00000100:00080000:2.0:1775765146.475654:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765146.475661:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765146.475705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861305984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765146.475731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765146.475734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765146.475736:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765146.475738:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 874 00000100:00080000:2.0:1775765146.475741:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765146.475744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765146.475753:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765146.475762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861306112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765146.475782:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861305984:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765146.475799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765146.475801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765146.475802:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765146.475807:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 874 00000100:00080000:2.0:1775765146.475811:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765146.475816:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765146.475822:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765146.475851:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861306240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765146.475864:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765146.475907:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765146/real 1775765146] req@ffffa0e9d0713100 x1862023861305984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765146.475923:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861305984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765146.475932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765146.475935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765146.475938:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861305984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765146.475960:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765146.475975:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765146.475983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765146/real 1775765146] req@ffffa0e9d0711880 x1862023861306112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765146.476000:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861306112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765146.476014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765146.476017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765146.476022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306112:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765146.476026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765146.476041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765146.476047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765146/real 1775765146] req@ffffa0e9d0712300 x1862023861306240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765146.476056:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861306240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765146.476061:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765146.476063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765146.476065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306240:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765147.051665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765147.307576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765148.075699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765148.332542:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765149.099613:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765149.355582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765150.123751:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765150.379630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765151.147653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765151.403562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765151.595565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765151.595576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765151.595582:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765151.595607:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 880 00000100:00080000:2.0:1775765151.595621:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765151.595629:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765151.595676:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861306368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765151.595707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765151.595710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765151.595712:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765151.595715:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 880 00000100:00080000:2.0:1775765151.595732:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765151.595738:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765151.595741:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861306368:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775765151.595751:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765151.595764:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861306496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765151.595773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765151.595776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765151.595778:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765151.595780:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 880 00000100:00080000:2.0:1775765151.595783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765151.595786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765151.595789:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765151.595797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861306624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765151.595818:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765151.595858:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765151/real 1775765151] req@ffffa0e9d0711500 x1862023861306368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765151.595881:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861306368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765151.595890:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765151.595894:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765151.595897:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861306368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765151.595921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765151.595940:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765151.595950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765151/real 1775765151] req@ffffa0e9d0713480 x1862023861306496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765151.595964:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861306496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765151.595973:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765151.595976:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765151.595980:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306496:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765151.595986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765151.596007:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765151.596019:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765151/real 1775765151] req@ffffa0e9d0712a00 x1862023861306624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765151.596029:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861306624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765151.596045:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765151.596047:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765151.596050:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306624:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765152.171615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765152.427584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765153.195703:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765153.451581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765154.219686:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765154.475625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765155.243740:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765155.499601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765156.267751:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765156.523634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765156.715583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765156.715599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765156.715605:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765156.715619:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 885 00000100:00080000:2.0:1775765156.715632:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765156.715654:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765156.715703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861306752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765156.715796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765156.715799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765156.715801:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765156.715805:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 885 00000100:00080000:2.0:1775765156.715810:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765156.715815:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765156.715826:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765156.715859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861306880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765156.715870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765156.715875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775765156.715876:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861306752:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765156.715877:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765156.715887:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 885 00000100:00080000:2.0:1775765156.715891:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765156.715894:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765156.715898:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765156.715944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861307008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765156.716007:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765156.716050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765156/real 1775765156] req@ffffa0e9d0710e00 x1862023861306752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765156.716079:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861306752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765156.716091:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765156.716096:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765156.716100:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861306752:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765156.716131:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765156.716156:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765156.716166:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765156/real 1775765156] req@ffffa0e9d0710a80 x1862023861306880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765156.716185:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861306880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765156.716197:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765156.716200:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765156.716203:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861306880:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765156.716207:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765156.716244:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765156.716257:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765156/real 1775765156] req@ffffa0e9d0710700 x1862023861307008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765156.716270:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861307008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765156.716279:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765156.716283:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765156.716287:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307008:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765157.291696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765157.547601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765158.315636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765158.571659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765159.339688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765159.595647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765160.363650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765160.619563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765161.387640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765161.643654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765161.835619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765161.835636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765161.835643:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765161.835659:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 890 00000100:00080000:2.0:1775765161.835691:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765161.835699:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765161.835739:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861307136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765161.835778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765161.835782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765161.835785:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765161.835789:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 890 00000100:00080000:2.0:1775765161.835793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765161.835797:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765161.835808:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765161.835822:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861307264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765161.835836:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765161.835840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765161.835844:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765161.835849:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 890 00000100:00080000:2.0:1775765161.835855:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765161.835863:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765161.835871:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861307136:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775765161.835873:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765161.835900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861307392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765161.836061:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765161.836114:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307392:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765162.411685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765162.667645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765163.435787:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765163.691638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765164.459665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765164.715654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765164.907793:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765164.907855:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765164.907882:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765164.907921:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765164.907949:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765164.907977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765161/real 1775765164] req@ffffa0e9d0712680 x1862023861307264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765164.908025:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861307264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765164.908038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765164.908044:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765164.908049:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765164.908059:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765161/real 1775765164] req@ffffa0e9d0712d80 x1862023861307392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765164.908074:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861307392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765164.908083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765164.908087:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765164.908091:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307392:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765164.908099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765161/real 1775765164] req@ffffa0e9d0711f80 x1862023861307136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765164.908112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861307136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765164.908119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765164.908128:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765164.908161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861307136:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775765165.483661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765165.739623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765166.507719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765166.763565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765166.955580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765166.955594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765166.955599:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765166.955609:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 895 00000100:00080000:2.0:1775765166.955620:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765166.955626:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765166.955659:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861307520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765166.955686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765166.955689:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765166.955690:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765166.955693:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 895 00000100:00080000:2.0:1775765166.955696:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765166.955699:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765166.955708:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765166.955717:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861307648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765166.955726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765166.955746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765166.955749:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765166.955750:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861307520:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765166.955758:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 895 00000100:00080000:2.0:1775765166.955762:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765166.955766:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765166.955788:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765166.955806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861307776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765166.955824:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765166.955869:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765166/real 1775765166] req@ffffa0e9d0711c00 x1862023861307520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765166.955892:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861307520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765166.955912:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765166.955917:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765166.955922:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861307520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765166.955951:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765166.955979:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765166.955993:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765166/real 1775765166] req@ffffa0e9d0713b80 x1862023861307648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765166.956005:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861307648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765166.956013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765166.956017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765166.956021:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307648:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765166.956028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765166.956062:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765166.956075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765166/real 1775765166] req@ffffa0e9d0710000 x1862023861307776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765166.956089:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861307776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765166.956098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765166.956102:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765166.956105:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861307776:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765167.531625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765167.787637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765168.555633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765168.811599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765169.579760:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765169.835645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765170.603609:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765170.859564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765171.627632:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765171.883658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765172.075765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765172.075782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765172.075789:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765172.075806:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 900 00000100:00080000:2.0:1775765172.075824:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765172.075835:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765172.075908:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861307904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765172.075956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765172.075962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765172.075964:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765172.075969:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 900 00000100:00080000:2.0:1775765172.075973:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765172.075978:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765172.075993:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765172.076010:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861308032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765172.076024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765172.076038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775765172.076039:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861307904:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765172.076055:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765172.076062:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 900 00000100:00080000:2.0:1775765172.076067:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765172.076073:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765172.076093:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765172.076114:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861308160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765172.076158:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765172.076207:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765172/real 1775765172] req@ffffa0e9d0710380 x1862023861307904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765172.076244:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861307904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765172.076254:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765172.076258:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765172.076262:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861307904:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765172.076304:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765172.076332:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765172.076342:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765172/real 1775765172] req@ffffa0e9d0711180 x1862023861308032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765172.076351:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861308032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765172.076357:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765172.076360:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765172.076362:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765172.076366:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765172.076377:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765172.076383:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765172/real 1775765172] req@ffffa0e9d0710000 x1862023861308160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765172.076391:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861308160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765172.076397:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765172.076400:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765172.076403:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308160:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765172.651695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765172.907549:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765173.675650:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765173.931641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765174.699644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765174.955607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765175.723730:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765175.979597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765176.747620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765177.003583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765177.195556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765177.195563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765177.195567:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765177.195575:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 905 00000100:00080000:2.0:1775765177.195584:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765177.195590:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765177.195626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861308288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765177.195655:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765177.195658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765177.195660:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765177.195663:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 905 00000100:00080000:2.0:1775765177.195666:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765177.195669:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765177.195678:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765177.195690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861308416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765177.195701:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861308288:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765177.195708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765177.195710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765177.195713:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765177.195720:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 905 00000100:00080000:2.0:1775765177.195723:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765177.195726:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765177.195729:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765177.195740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861308544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765177.195771:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765177.195803:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765177/real 1775765177] req@ffffa0e9d0713b80 x1862023861308288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765177.195829:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861308288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765177.195837:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765177.195841:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765177.195845:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861308288:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765177.195865:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765177.195888:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765177.195902:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765177/real 1775765177] req@ffffa0e9d0711c00 x1862023861308416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765177.195916:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861308416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765177.195925:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765177.195927:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765177.195938:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308416:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765177.195943:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765177.195959:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765177.195967:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765177/real 1775765177] req@ffffa0e9d0711f80 x1862023861308544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765177.195986:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861308544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765177.195991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765177.195993:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765177.195995:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308544:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765177.771637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765178.027583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765178.795663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765179.051624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765179.819596:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765180.075599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765180.843658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765181.099636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765181.867649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765182.123636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765182.315577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765182.315585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765182.315589:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765182.315597:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 910 00000100:00080000:2.0:1775765182.315607:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765182.315612:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765182.315647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861308672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765182.315670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765182.315673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765182.315674:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765182.315677:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 910 00000100:00080000:2.0:1775765182.315693:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765182.315697:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765182.315706:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765182.315724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861308800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765182.315727:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861308672:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765182.315749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765182.315752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765182.315753:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765182.315756:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 910 00000100:00080000:2.0:1775765182.315759:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765182.315762:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765182.315766:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765182.315778:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861308928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765182.315823:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765182.315862:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765182/real 1775765182] req@ffffa0e9d0712d80 x1862023861308672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765182.315879:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861308672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765182.315891:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765182.315896:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765182.315901:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861308672:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765182.315930:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765182.315955:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765182.315965:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765182/real 1775765182] req@ffffa0e9d0712680 x1862023861308800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765182.316001:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861308800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765182.316011:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765182.316015:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765182.316019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308800:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765182.316026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765182.316051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765182.316061:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765182/real 1775765182] req@ffffa0e9d0710700 x1862023861308928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765182.316070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861308928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765182.316075:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765182.316078:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765182.316080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861308928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765182.891633:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765183.147635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765183.915654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765184.171591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765184.939646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765185.195622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765185.963586:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765186.219660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765186.987747:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765187.243673:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765187.435709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765187.435726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765187.435734:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765187.435752:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 915 00000100:00080000:2.0:1775765187.435771:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765187.435782:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765187.435843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861309056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765187.435914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765187.435921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765187.435925:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765187.435933:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 915 00000100:00080000:2.0:1775765187.435964:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765187.435973:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765187.435991:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765187.436010:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861309056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765187.436023:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861309184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765187.436046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765187.436050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765187.436053:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765187.436058:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 915 00000100:00080000:2.0:1775765187.436064:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765187.436069:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765187.436076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765187.436095:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861309312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765187.436122:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765187.436175:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765187/real 1775765187] req@ffffa0e9d0710a80 x1862023861309056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765187.436211:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861309056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765187.436225:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765187.436230:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765187.436236:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861309056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765187.436273:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765187.436316:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765187.436340:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765187/real 1775765187] req@ffffa0e9d0710e00 x1862023861309184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765187.436362:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861309184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765187.436372:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765187.436376:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765187.436380:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309184:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765187.436392:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765187.436426:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765187.436446:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765187/real 1775765187] req@ffffa0e9d0712a00 x1862023861309312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765187.436469:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861309312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765187.436577:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765187.436585:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765187.436591:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309312:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765188.011713:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765188.267587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765189.035700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765189.291660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765190.059641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765190.315586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765191.083786:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765191.339567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765192.107754:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765192.363673:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765192.555719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765192.555740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765192.555749:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765192.555766:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 921 00000100:00080000:2.0:1775765192.555813:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765192.555824:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765192.555883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861309440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765192.555987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765192.555995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765192.556005:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765192.556015:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 921 00000100:00080000:2.0:1775765192.556025:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765192.556035:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765192.556055:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765192.556088:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861309440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765192.556106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861309568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765192.556126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765192.556131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765192.556134:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765192.556139:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 921 00000100:00080000:2.0:1775765192.556146:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765192.556151:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765192.556159:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765192.556175:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861309696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765192.556218:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765192.556281:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765192/real 1775765192] req@ffffa0e9d0713480 x1862023861309440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765192.556318:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861309440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765192.556333:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765192.556339:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765192.556345:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861309440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765192.556386:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765192.556430:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765192.556457:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765192/real 1775765192] req@ffffa0e9d0711500 x1862023861309568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765192.556540:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861309568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765192.556568:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765192.556573:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765192.556578:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309568:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765192.556587:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765192.556620:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765192.556636:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765192/real 1775765192] req@ffffa0e9d0712300 x1862023861309696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765192.556651:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861309696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765192.556661:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765192.556665:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765192.556669:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309696:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765193.131702:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765193.387597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765194.155723:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765194.411588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765195.179660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765195.435601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765196.203745:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765196.459586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765197.227645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765197.483609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765197.675778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765197.675799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765197.675809:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765197.675831:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 926 00000100:00080000:0.0:1775765197.675881:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765197.675895:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765197.675985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861309824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765197.676046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765197.676053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765197.676057:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765197.676067:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 926 00000100:00080000:0.0:1775765197.676078:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765197.676087:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765197.676097:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861309824:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775765197.676136:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765197.676171:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765197.676190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861309952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765197.676216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765197.676224:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765197.676228:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765197.676236:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765197/real 1775765197] req@ffffa0e9d09ddf80 x1862023861309824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765197.676247:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 926 00000100:00080000:0.0:1775765197.676271:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765197.676279:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861309824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765197.676287:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00080000:2.0:1775765197.676307:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775765197.676310:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765197.676314:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765197.676326:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861309824:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765197.676347:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861310080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765197.676368:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765197.676426:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765197.676447:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765197/real 1775765197] req@ffffa0e9d09dca80 x1862023861309952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765197.676464:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861309952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765197.676473:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765197.676478:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765197.676527:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861309952:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765197.676560:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765197.676593:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765197.676608:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765197/real 1775765197] req@ffffa0e9d09de680 x1862023861310080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765197.676632:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861310080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765197.676639:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765197.676643:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765197.676647:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765198.251625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765198.507598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765199.275631:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765199.531565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765200.299734:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765200.555620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765201.323594:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765201.579565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765202.347623:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765202.603582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765202.795771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765202.795780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765202.795785:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765202.795813:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 931 00000100:00080000:2.0:1775765202.795842:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765202.795850:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765202.795896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861310208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765202.795929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765202.795934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765202.795936:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765202.795940:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 931 00000100:00080000:2.0:1775765202.795945:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765202.795951:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765202.795963:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765202.795973:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861310208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765202.796021:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861310336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765202.796031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765202.796035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765202.796037:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765202.796040:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 931 00000100:00080000:2.0:1775765202.796044:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765202.796051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:2.0:1775765202.796051:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765202.796061:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765202.796077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861310464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765202.796105:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765202/real 1775765202] req@ffffa0e9d0711880 x1862023861310208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765202.796124:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861310208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765202.796133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765202.796137:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765202.796140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861310208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765202.796162:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765202.796181:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765202.796191:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765202/real 1775765202] req@ffffa0e9d0713100 x1862023861310336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765202.796200:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861310336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765202.796207:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765202.796210:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765202.796212:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765202.796217:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765202.796229:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765202.796239:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765202/real 1775765202] req@ffffa0e9d0712300 x1862023861310464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765202.796265:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861310464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765202.796275:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765202.796280:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765202.796284:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310464:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765203.371662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765203.627570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765204.395630:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765204.651586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765205.419599:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765205.675578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765206.443638:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765206.699587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765207.467634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765207.723607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765207.915734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765207.915762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765207.915768:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765207.915781:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 936 00000100:00080000:2.0:1775765207.915795:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765207.915805:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765207.915876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861310592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765207.915926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765207.915931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765207.915935:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765207.915942:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 936 00000100:00080000:2.0:1775765207.915965:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765207.915968:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861310592:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765207.915977:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765207.915993:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765207.916017:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861310720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765207.916036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765207.916041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765207.916044:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765207.916050:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 936 00000100:00080000:2.0:1775765207.916056:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765207.916061:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765207.916068:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765207.916086:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765207.916089:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861310848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765207.916142:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765207/real 1775765207] req@ffffa0e9d0711500 x1862023861310592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765207.916164:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861310592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765207.916175:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765207.916179:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765207.916183:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861310592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765207.916209:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765207.916235:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765207.916246:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765207/real 1775765207] req@ffffa0e9d0713480 x1862023861310720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765207.916268:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861310720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765207.916276:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765207.916279:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765207.916282:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310720:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765207.916287:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765207.916306:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765207.916315:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765207/real 1775765207] req@ffffa0e9d0712a00 x1862023861310848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765207.916326:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861310848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765207.916333:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765207.916335:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765207.916338:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861310848:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765208.491668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765208.747599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765209.515653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765209.771556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765210.539719:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765210.795576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765211.563624:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765211.819557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765212.587653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765212.843612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765213.035571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765213.035581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765213.035587:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765213.035598:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 941 00000100:00080000:2.0:1775765213.035611:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765213.035619:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765213.035665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861310976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765213.035746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765213.035750:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765213.035752:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765213.035757:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 941 00000100:00080000:2.0:1775765213.035762:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765213.035767:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765213.035777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765213.035790:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861310976:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765213.035791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861311104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765213.035802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765213.035805:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765213.035807:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765213.035811:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 941 00000100:00080000:2.0:1775765213.035814:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765213.035817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765213.035822:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765213.035830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861311232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765213.035850:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765213.035888:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765213/real 1775765213] req@ffffa0e9d0710e00 x1862023861310976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765213.035909:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861310976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765213.035916:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765213.035919:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765213.035922:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861310976:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765213.035941:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765213.035959:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765213.035967:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765213/real 1775765213] req@ffffa0e9d0710a80 x1862023861311104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765213.035976:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861311104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765213.035981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765213.035983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765213.035985:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311104:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765213.035989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765213.036014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765213.036031:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765213/real 1775765213] req@ffffa0e9d0710700 x1862023861311232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765213.036043:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861311232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765213.036064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765213.036068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765213.036071:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311232:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765213.611676:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765213.867651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765214.635727:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765214.891618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765215.659646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765215.915555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765216.683614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765216.939575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765217.707756:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765217.963572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765218.155775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765218.155787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765218.155793:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765218.155824:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 946 00000100:00080000:2.0:1775765218.155836:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765218.155844:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765218.155883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861311360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765218.155924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765218.155927:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765218.155929:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765218.155932:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 946 00000100:00080000:2.0:1775765218.155936:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765218.155939:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765218.155948:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765218.155959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861311488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765218.155969:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765218.155973:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861311360:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765218.155973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765218.155975:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765218.155983:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 946 00000100:00080000:2.0:1775765218.155986:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765218.155988:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765218.155991:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765218.156001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861311616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765218.156038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765218.156091:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765218/real 1775765218] req@ffffa0e9d0712680 x1862023861311360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765218.156106:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861311360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765218.156114:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765218.156118:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765218.156121:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861311360:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765218.156138:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765218.156154:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765218.156163:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765218/real 1775765218] req@ffffa0e9d0712d80 x1862023861311488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765218.156171:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861311488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765218.156188:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765218.156190:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765218.156193:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311488:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765218.156197:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765218.156212:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765218.156222:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765218/real 1775765218] req@ffffa0e9d0711f80 x1862023861311616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765218.156234:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861311616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765218.156242:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765218.156245:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765218.156249:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311616:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765218.731677:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765218.987564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765219.755697:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765220.011598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765220.779607:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765221.035671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765221.803739:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765222.059639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765222.827699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765223.083642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765223.275744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765223.275757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765223.275763:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765223.275777:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 951 00000100:00080000:0.0:1775765223.275792:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765223.275800:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765223.275862:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861311744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765223.275962:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861311744:192.168.202.104@tcp:250:kworker.0 00000800:00000100:2.0:1775765223.276071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775765223.276109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765223.276113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765223.276115:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765223.276119:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 951 00000100:00080000:0.0:1775765223.276124:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765223.276128:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765223.276138:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765223.276154:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861311872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765223.276214:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765223/real 1775765223] req@ffffa0e9d09dd500 x1862023861311744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765223.276232:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861311744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765223.276241:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765223.276244:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765223.276248:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861311744:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765223.276271:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765223.276301:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765223.276319:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765223/real 1775765223] req@ffffa0e9d09ddc00 x1862023861311872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765223.276337:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861311872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765223.276349:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765223.276353:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765223.276359:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861311872:192.168.202.104@tcp:38:kworker.0 00000100:00080000:0.0:1775765223.276421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765223.276425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765223.276427:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765223.276431:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 951 00000100:00080000:0.0:1775765223.276436:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765223.276440:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765223.276446:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765223.276459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861312000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765223.276568:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765223.276626:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765223.276640:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765223/real 1775765223] req@ffffa0e9d09dd180 x1862023861312000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765223.276651:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861312000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765223.276658:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765223.276661:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765223.276664:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312000:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765223.851811:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765224.107662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765224.875680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765225.131656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765225.899629:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765226.155758:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765226.923771:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765227.179655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765227.947645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765228.203636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765228.395546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765228.395565:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765228.395570:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765228.395580:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 956 00000100:00080000:0.0:1775765228.395590:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765228.395598:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765228.395636:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861312128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765228.395662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765228.395666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765228.395668:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765228.395673:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 956 00000100:00080000:0.0:1775765228.395678:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765228.395682:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765228.395690:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765228.395698:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861312128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765228.395704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861312256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765228.395713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765228.395715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765228.395716:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765228.395719:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 956 00000100:00080000:0.0:1775765228.395722:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765228.395724:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765228.395737:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765228.395746:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861312384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765228.395805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765228.395830:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312384:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765228.971650:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765229.227748:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765229.995671:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765230.251659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765231.019800:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765231.275727:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765231.468001:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765231.468089:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765231.468114:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765231.468172:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765231.468184:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765231.468234:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765228/real 1775765231] req@ffffa0e9d09df800 x1862023861312256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765231.468273:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861312256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765231.468295:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765231.468304:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765231.468312:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765231.468352:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765228/real 1775765231] req@ffffa0e9d09dc700 x1862023861312384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765231.468379:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861312384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765231.468395:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765231.468402:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765231.468408:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312384:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765231.468421:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765228/real 1775765231] req@ffffa0e9d09dd880 x1862023861312128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765231.468444:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861312128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765231.468458:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765231.468464:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765231.468470:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861312128:192.168.202.104@tcp:250:kworker.0 00010000:00010000:3.0:1775765232.043631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765232.299592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765233.067634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765233.323567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765233.516629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765233.516637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765233.516641:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765233.516649:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 962 00000100:00080000:2.0:1775765233.516658:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765233.516668:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765233.516700:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861312512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765233.516734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765233.516738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765233.516740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765233.516744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 962 00000100:00080000:2.0:1775765233.516748:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765233.516753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765233.516763:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765233.516773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861312512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765233.516776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861312640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765233.516785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765233.516789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765233.516790:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765233.516793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 962 00000100:00080000:2.0:1775765233.516796:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765233.516800:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765233.516804:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765233.516812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861312768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765233.516831:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765233.516878:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765233/real 1775765233] req@ffffa0e9d0711c00 x1862023861312512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765233.516893:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861312512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765233.516900:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765233.516903:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765233.516906:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861312512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765233.516925:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765233.516941:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765233.516950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765233/real 1775765233] req@ffffa0e9d0713b80 x1862023861312640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765233.516958:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861312640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765233.516980:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765233.516984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765233.516987:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312640:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765233.516993:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765233.517014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765233.517025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765233/real 1775765233] req@ffffa0e9d0710000 x1862023861312768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765233.517034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861312768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765233.517041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765233.517044:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765233.517047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861312768:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765234.091603:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765234.347591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765235.115710:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765235.371643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765236.139663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765236.395615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765237.163752:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765237.419623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765238.187574:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765238.443582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765238.635616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765238.635624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765238.635629:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765238.635636:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 967 00000100:00080000:0.0:1775765238.635645:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765238.635652:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765238.635710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861312896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765238.635736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765238.635739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765238.635740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765238.635743:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 967 00000100:00080000:0.0:1775765238.635747:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765238.635751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765238.635760:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765238.635767:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861312896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765238.635772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861313024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765238.635781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765238.635783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765238.635784:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765238.635787:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 967 00000100:00080000:0.0:1775765238.635790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765238.635792:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765238.635795:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765238.635803:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861313152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765238.635830:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765238.635871:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765238/real 1775765238] req@ffffa0e9d09dfb80 x1862023861312896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765238.635901:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861312896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765238.635917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765238.635921:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765238.635924:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861312896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765238.635945:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765238.635979:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765238.635992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765238/real 1775765238] req@ffffa0e9d09df100 x1862023861313024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765238.636007:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861313024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765238.636014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765238.636017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765238.636019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765238.636024:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765238.636040:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765238.636047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765238/real 1775765238] req@ffffa0e9d09ded80 x1862023861313152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765238.636055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861313152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765238.636071:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765238.636073:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765238.636075:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313152:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765239.211625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765239.467629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765240.235635:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765240.491597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765241.259614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765241.515666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765242.283602:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765242.539559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765243.307701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765243.563601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765243.755861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765243.755874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765243.755880:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765243.755893:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 972 00000100:00080000:0.0:1775765243.755908:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765243.755916:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765243.755984:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861313280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765243.756028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765243.756034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765243.756037:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765243.756044:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 972 00000100:00080000:0.0:1775765243.756051:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765243.756057:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765243.756072:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765243.756096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861313408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765243.756103:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861313280:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765243.756129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765243.756135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765243.756138:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765243.756146:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 972 00000100:00080000:0.0:1775765243.756153:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765243.756160:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765243.756183:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765243.756201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861313536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765243.756210:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765243.756290:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765243/real 1775765243] req@ffffa0e9d09dc380 x1862023861313280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765243.756324:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861313280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765243.756336:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765243.756341:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765243.756345:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861313280:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765243.756374:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765243.756401:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765243.756439:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765243/real 1775765243] req@ffffa0e9d09dc000 x1862023861313408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765243.756456:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861313408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765243.756470:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765243.756535:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765243.756543:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313408:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765243.756557:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765243.756588:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765243.756601:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765243/real 1775765243] req@ffffa0e9d09dce00 x1862023861313536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765243.756623:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861313536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765243.756631:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765243.756635:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765243.756638:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313536:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765244.331666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765244.587689:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765245.355641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765245.611613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765246.379667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765246.635569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765247.403712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765247.659627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765248.427684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765248.683601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765248.875732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765248.875753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765248.875760:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765248.875773:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 977 00000100:00080000:0.0:1775765248.875788:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765248.875799:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765248.875847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861313664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765248.875887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765248.875893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765248.875896:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765248.875901:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 977 00000100:00080000:0.0:1775765248.875906:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765248.875911:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765248.875926:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765248.875946:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861313664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765248.875951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861313792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765248.876022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765248.876026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765248.876029:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765248.876037:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775765248.876040:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 977 00000100:00080000:0.0:1775765248.876049:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765248.876054:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765248.876061:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765248.876084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861313920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765248.876097:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765248/real 1775765248] req@ffffa0e9d09de300 x1862023861313664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765248.876121:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861313664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765248.876132:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765248.876137:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765248.876141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861313664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765248.876188:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765248.876225:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765248.876244:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765248/real 1775765248] req@ffffa0e9d09dea00 x1862023861313792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765248.876263:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861313792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765248.876271:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765248.876274:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765248.876278:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313792:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765248.876283:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765248.876305:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765248.876337:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765248/real 1775765248] req@ffffa0e9d09df480 x1862023861313920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765248.876348:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861313920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765248.876356:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765248.876359:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765248.876361:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861313920:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765249.451717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765249.707595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765250.475588:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765250.731647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765251.499647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765251.755577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765252.523613:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765252.779592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765253.547568:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765253.803591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765253.995606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765253.995618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765253.995642:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765253.995651:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 982 00000100:00080000:2.0:1775765253.995661:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765253.995667:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765253.995702:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861314048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765253.995731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765253.995734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765253.995735:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765253.995738:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 982 00000100:00080000:2.0:1775765253.995741:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765253.995744:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765253.995752:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765253.995763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861314176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765253.995772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765253.995775:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861314048:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765253.995776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765253.995778:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765253.995787:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 982 00000100:00080000:2.0:1775765253.995789:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765253.995792:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765253.995795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765253.995805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861314304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765253.995854:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765253.995902:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765253/real 1775765253] req@ffffa0e9d0711180 x1862023861314048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765253.995922:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861314048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765253.995931:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765253.995935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765253.995939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861314048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765253.995976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765253.996005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765253.996019:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765253/real 1775765253] req@ffffa0e9d0710380 x1862023861314176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765253.996029:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861314176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765253.996036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765253.996038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765253.996041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765253.996045:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765253.996060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765253.996066:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765253/real 1775765253] req@ffffa0e9d0710000 x1862023861314304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765253.996075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861314304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765253.996080:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765253.996083:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765253.996085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314304:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765254.571640:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765254.827567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765255.595618:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765255.851605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765256.619575:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765256.875675:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765257.643644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765257.899657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765258.667737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765258.923563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765259.115662:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765259.115674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765259.115680:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765259.115692:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 987 00000100:00080000:2.0:1775765259.115705:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765259.115714:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765259.115769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861314432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765259.115828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765259.115845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765259.115848:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765259.115853:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 987 00000100:00080000:2.0:1775765259.115859:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765259.115864:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765259.115876:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765259.115894:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861314560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765259.115909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765259.115915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765259.115918:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765259.115923:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 987 00000100:00080000:2.0:1775765259.115947:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765259.115953:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765259.115965:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765259.115967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861314432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765259.115984:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861314688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765259.116079:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765259.116135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765259/real 1775765259] req@ffffa0e9d0713b80 x1862023861314432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765259.116156:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861314432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765259.116168:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765259.116173:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765259.116177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861314432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765259.116184:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765259.116204:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765259.116215:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765259/real 1775765259] req@ffffa0e9d0711c00 x1862023861314560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765259.116225:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861314560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765259.116232:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765259.116235:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765259.116237:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314560:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765259.116299:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765259.116330:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765259.116346:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765259/real 1775765259] req@ffffa0e9d0711f80 x1862023861314688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765259.116379:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861314688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765259.116387:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765259.116390:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765259.116393:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314688:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765259.691606:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765259.947666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765260.715735:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765260.971657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765261.739673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765261.995676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765262.763712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765263.019616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765263.787749:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765264.043620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765264.235659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765264.235674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765264.235681:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765264.235695:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 992 00000100:00080000:1.0:1775765264.235712:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765264.235720:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765264.235778:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861314816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765264.235815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765264.235820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765264.235822:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765264.235826:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 992 00000100:00080000:1.0:1775765264.235829:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765264.235833:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765264.235846:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765264.235858:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861314944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765264.235909:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861314816:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765264.235913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765264.235920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765264.235923:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765264.235928:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 992 00000100:00080000:1.0:1775765264.235933:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765264.235937:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765264.235942:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765264.235957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861315072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765264.236009:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765264.236064:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765264/real 1775765264] req@ffffa0e9c8dec380 x1862023861314816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765264.236086:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861314816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765264.236098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765264.236103:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765264.236107:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861314816:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765264.236134:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765264.236157:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765264.236169:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765264/real 1775765264] req@ffffa0e9c8deed80 x1862023861314944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765264.236193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861314944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765264.236202:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765264.236205:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765264.236208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861314944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765264.236243:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765264.236277:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765264.236295:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765264/real 1775765264] req@ffffa0e9c8ded500 x1862023861315072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765264.236315:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861315072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765264.236326:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765264.236341:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765264.236344:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315072:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765264.811661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765265.067583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765265.835662:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765266.091691:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765266.859716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765267.115607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765267.883707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765268.139692:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765268.907598:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765269.163732:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765269.355566:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765269.355577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765269.355582:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765269.355592:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 997 00000100:00080000:1.0:1775765269.355603:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765269.355610:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765269.355644:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861315200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765269.355668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765269.355670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765269.355672:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765269.355675:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 997 00000100:00080000:1.0:1775765269.355692:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765269.355696:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765269.355705:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765269.355717:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861315328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765269.355718:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861315200:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765269.355733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765269.355735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765269.355736:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765269.355739:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 997 00000100:00080000:1.0:1775765269.355741:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765269.355744:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765269.355747:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765269.355756:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861315456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765269.355785:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765269.355828:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765269/real 1775765269] req@ffffa0e9c8def480 x1862023861315200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765269.355850:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861315200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765269.355858:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765269.355861:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765269.355864:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861315200:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765269.355885:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765269.355909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765269.355922:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765269/real 1775765269] req@ffffa0e9c8dedc00 x1862023861315328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765269.355935:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861315328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765269.355943:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765269.355946:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765269.355949:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315328:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765269.355953:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765269.355968:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765269.355975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765269/real 1775765269] req@ffffa0e9c8def800 x1862023861315456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765269.355983:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861315456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765269.355997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765269.355999:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765269.356001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315456:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765269.931605:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765270.187618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765270.955628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765271.211683:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765271.979632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765272.235665:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765273.003741:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765273.259619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765274.027663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765274.283645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765274.475673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765274.475685:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765274.475703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765274.475733:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1002 00000100:00080000:2.0:1775765274.475745:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765274.475754:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765274.475799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861315584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765274.475836:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765274.475840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765274.475849:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765274.475854:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1002 00000100:00080000:2.0:1775765274.475858:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765274.475862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765274.475874:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765274.475912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861315712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765274.475921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861315584:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765274.475950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765274.475957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765274.475959:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765274.475984:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1002 00000100:00080000:2.0:1775765274.475993:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765274.476013:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775765274.476015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:2.0:1775765274.476031:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765274.476053:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861315840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765274.476095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765274/real 1775765274] req@ffffa0e9d0712d80 x1862023861315584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765274.476151:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861315584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765274.476172:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765274.476180:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765274.476188:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861315584:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765274.476223:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765274.476256:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765274.476271:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765274/real 1775765274] req@ffffa0e9d0712680 x1862023861315712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765274.476287:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861315712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765274.476315:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765274.476322:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765274.476328:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315712:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765274.476340:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765274.476421:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765274.476445:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765274/real 1775765274] req@ffffa0e9d0710700 x1862023861315840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765274.476461:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861315840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765274.476472:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765274.476476:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765274.476480:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861315840:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765275.051602:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765275.307644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765276.075730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765276.331620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765277.099610:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765277.355605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765278.123718:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765278.379665:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765279.147722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765279.403665:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765279.595647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765279.595662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765279.595670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765279.595684:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1008 00000100:00080000:1.0:1775765279.595702:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765279.595712:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765279.595767:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861315968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765279.595835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765279.595842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765279.595846:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765279.595854:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1008 00000100:00080000:1.0:1775765279.595863:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765279.595870:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765279.595886:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765279.595912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861316096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765279.595928:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861315968:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765279.595942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765279.595950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765279.595954:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765279.595964:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1008 00000100:00080000:1.0:1775765279.595974:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765279.595981:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765279.596000:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765279.596025:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861316224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765279.596031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765279.596156:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765279/real 1775765279] req@ffffa0e9c8ded880 x1862023861315968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765279.596195:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861315968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765279.596231:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765279.596237:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765279.596243:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861315968:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765279.596271:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765279.596303:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765279.596330:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765279/real 1775765279] req@ffffa0e9c8deea00 x1862023861316096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765279.596344:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861316096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765279.596354:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765279.596357:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765279.596361:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316096:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765279.596367:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316224:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765279.596385:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765279.596395:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765279/real 1775765279] req@ffffa0e9c8dee680 x1862023861316224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765279.596408:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861316224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765279.596416:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765279.596419:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765279.596423:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316224:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765280.171667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765280.427688:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765281.195737:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765281.451606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765282.219795:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765282.475619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765283.243685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765283.499597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765284.267628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765284.523654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765284.715672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765284.715684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765284.715690:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765284.715703:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1013 00000100:00080000:0.0:1775765284.715717:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765284.715725:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765284.715781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861316352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765284.715819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765284.715823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765284.715825:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765284.715828:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1013 00000100:00080000:0.0:1775765284.715832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765284.715835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765284.715848:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765284.715860:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861316480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765284.715881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765284.715886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765284.715892:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861316352:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765284.715916:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765284.715922:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1013 00000100:00080000:0.0:1775765284.715927:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765284.715931:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765284.715937:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765284.715952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861316608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765284.715986:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765284.716053:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765284/real 1775765284] req@ffffa0e9d09df480 x1862023861316352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765284.716069:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861316352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765284.716077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765284.716080:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765284.716083:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861316352:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765284.716109:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765284.716129:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765284.716138:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765284/real 1775765284] req@ffffa0e9d09dea00 x1862023861316480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765284.716146:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861316480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765284.716152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765284.716154:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765284.716156:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316480:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765284.716175:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765284.716186:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765284.716192:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765284/real 1775765284] req@ffffa0e9d09de300 x1862023861316608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765284.716200:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861316608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765284.716204:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765284.716206:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765284.716208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316608:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765285.291631:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765285.547599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765286.315709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765286.571722:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765287.339581:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765287.595634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765288.363636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765288.619571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765289.387668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765289.643654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765289.835674:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765289.835721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765289.835726:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765289.835737:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1018 00000100:00080000:2.0:1775765289.835751:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765289.835774:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765289.835804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861316736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765289.835830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765289.835832:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765289.835834:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765289.835836:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1018 00000100:00080000:2.0:1775765289.835839:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765289.835842:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765289.835851:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765289.835862:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861316864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765289.835863:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861316736:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765289.835877:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765289.835879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765289.835880:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765289.835882:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1018 00000100:00080000:2.0:1775765289.835884:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765289.835886:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765289.835889:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765289.835896:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861316992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765289.835944:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765289.836022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765289/real 1775765289] req@ffffa0e9d0710a80 x1862023861316736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765289.836045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861316736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765289.836054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765289.836057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765289.836060:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861316736:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765289.836078:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765289.836097:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765289.836106:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765289/real 1775765289] req@ffffa0e9d0710e00 x1862023861316864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765289.836122:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861316864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765289.836128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765289.836130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765289.836132:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316864:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765289.836135:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765289.836148:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765289.836154:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765289/real 1775765289] req@ffffa0e9d0712a00 x1862023861316992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765289.836162:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861316992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765289.836167:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765289.836169:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765289.836171:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861316992:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765290.411578:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765290.667590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765291.435722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765291.691540:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765292.459697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765292.715603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765293.483614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765293.739588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765294.507696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765294.763636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765294.955627:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765294.955640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765294.955645:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765294.955658:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1023 00000100:00080000:1.0:1775765294.955671:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765294.955680:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765294.955746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861317120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765294.955802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765294.955810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765294.955814:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765294.955822:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1023 00000100:00080000:1.0:1775765294.955831:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765294.955840:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765294.955857:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765294.955866:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861317120:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765294.955887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861317248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765294.955905:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765294.955910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765294.955913:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765294.955918:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1023 00000100:00080000:1.0:1775765294.955961:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765294.955968:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765294.955976:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765294.956032:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861317376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765294.956035:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317248:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765294.956120:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317376:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765295.531727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765295.787673:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765296.555617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765296.811590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765297.579720:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765297.835598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765298.027823:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765298.027866:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765298.027898:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765298.027966:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765298.027981:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765298.028053:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765294/real 1775765298] req@ffffa0e9c8deca80 x1862023861317376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765298.028087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861317376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765298.028099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765298.028103:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765298.028107:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317376:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765298.028120:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765294/real 1775765298] req@ffffa0e9c8dedf80 x1862023861317248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765298.028129:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861317248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765298.028135:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765298.028138:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765298.028140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317248:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765298.028146:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765294/real 1775765298] req@ffffa0e9c8dece00 x1862023861317120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765298.028153:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861317120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765298.028180:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765298.028184:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765298.028188:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861317120:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775765298.603701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765298.859610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765299.627630:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765299.883599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765300.075598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765300.075610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765300.075616:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765300.075628:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1028 00000100:00080000:1.0:1775765300.075643:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765300.075651:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765300.075698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861317504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765300.075730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765300.075734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765300.075736:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765300.075740:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1028 00000100:00080000:1.0:1775765300.075743:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765300.075746:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765300.075758:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765300.075769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861317632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765300.075779:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765300.075782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765300.075784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765300.075809:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861317504:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765300.075826:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1028 00000100:00080000:1.0:1775765300.075830:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765300.075833:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765300.075839:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765300.075854:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861317760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765300.075972:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765300.076049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765300/real 1775765300] req@ffffa0e9c8def100 x1862023861317504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765300.076081:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861317504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765300.076098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765300.076105:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765300.076110:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861317504:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765300.076120:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765300.076156:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765300.076174:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765300/real 1775765300] req@ffffa0e9c8dec700 x1862023861317632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765300.076190:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861317632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765300.076200:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765300.076216:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765300.076221:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317632:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765300.076262:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765300.076293:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765300.076311:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765300/real 1775765300] req@ffffa0e9c8defb80 x1862023861317760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765300.076327:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861317760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765300.076337:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765300.076341:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765300.076345:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861317760:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765300.651789:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765300.907589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765301.675716:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765301.931675:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765302.699618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765302.955560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765303.723752:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765303.979614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765304.747709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765305.003636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765305.195631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765305.195642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765305.195648:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765305.195658:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1033 00000100:00080000:0.0:1775765305.195670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765305.195677:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765305.195722:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861317888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765305.195776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765305.195779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765305.195781:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765305.195784:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1033 00000100:00080000:0.0:1775765305.195787:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765305.195790:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765305.195800:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765305.195810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861318016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765305.195817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765305.195820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765305.195822:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765305.195826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1033 00000100:00080000:0.0:1775765305.195829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765305.195833:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765305.195838:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765305.195864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861318144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765305.195876:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861317888:192.168.202.104@tcp:250:kworker.0 00000800:00000100:2.0:1775765305.195960:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765305.196002:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765305/real 1775765305] req@ffffa0e9d09dce00 x1862023861317888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765305.196018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861317888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765305.196027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765305.196031:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765305.196034:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861317888:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765305.196040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765305.196057:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765305.196066:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765305/real 1775765305] req@ffffa0e9d09dc000 x1862023861318016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765305.196074:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861318016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765305.196080:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765305.196082:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765305.196099:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318016:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765305.196121:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765305.196137:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765305.196145:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765305/real 1775765305] req@ffffa0e9d09dc380 x1862023861318144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765305.196153:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861318144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765305.196159:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765305.196161:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765305.196163:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318144:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765305.771641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765306.027612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765306.795672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765307.051587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765307.819660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765308.075596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765308.843685:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765309.099634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765309.867709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765310.123658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765310.315598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765310.315607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765310.315612:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765310.315621:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1038 00000100:00080000:0.0:1775765310.315631:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765310.315638:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765310.315673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861318272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765310.315695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765310.315697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765310.315699:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765310.315702:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1038 00000100:00080000:0.0:1775765310.315704:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765310.315706:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765310.315714:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765310.315721:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861318400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765310.315729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765310.315732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765310.315734:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765310.315736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1038 00000100:00080000:0.0:1775765310.315738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765310.315740:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765310.315743:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765310.315752:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861318272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765310.315769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861318528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765310.315855:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765310.315900:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765310/real 1775765310] req@ffffa0e9d09ded80 x1862023861318272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765310.315916:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861318272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765310.315925:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765310.315928:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765310.315932:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861318272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765310.315938:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765310.315963:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765310.315972:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765310/real 1775765310] req@ffffa0e9d09df100 x1862023861318400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765310.315980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861318400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765310.315986:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765310.315988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765310.315990:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318400:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765310.316008:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765310.316019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765310.316025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765310/real 1775765310] req@ffffa0e9d09dfb80 x1862023861318528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765310.316032:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861318528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765310.316037:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765310.316039:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765310.316041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318528:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765310.891674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765311.147595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765311.915714:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765312.171623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765312.939729:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765313.195638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765313.963609:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765314.219581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765314.987842:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765315.243742:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765315.435631:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765315.435641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765315.435646:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765315.435657:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1043 00000100:00080000:1.0:1775765315.435670:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765315.435678:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765315.435721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861318656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765315.435764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765315.435768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765315.435770:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765315.435773:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1043 00000100:00080000:1.0:1775765315.435777:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765315.435780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765315.435803:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765315.435818:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861318656:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765315.435819:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861318784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765315.435834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765315.435837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765315.435838:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765315.435842:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1043 00000100:00080000:1.0:1775765315.435846:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765315.435849:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765315.435854:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765315.435874:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861318912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765315.435896:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765315.435942:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765315/real 1775765315] req@ffffa0e9c8defb80 x1862023861318656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765315.435968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861318656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765315.435977:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765315.435981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765315.435984:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861318656:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765315.436005:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765315.436024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765315.436054:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765315/real 1775765315] req@ffffa0e9c8dec700 x1862023861318784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765315.436065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861318784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765315.436072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765315.436075:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765315.436078:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318784:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765315.436083:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765315.436097:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765315.436104:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765315/real 1775765315] req@ffffa0e9c8def100 x1862023861318912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765315.436121:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861318912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765315.436140:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765315.436143:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765315.436161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861318912:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765316.011629:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765316.267564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765317.035625:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765317.291592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765318.059665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765318.315596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765319.083808:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765319.339687:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765320.107764:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765320.363646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765320.555632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765320.555647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765320.555655:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765320.555669:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1049 00000100:00080000:1.0:1775765320.555687:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765320.555697:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765320.555790:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861319040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765320.555848:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765320.555856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765320.555887:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765320.555897:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1049 00000100:00080000:1.0:1775765320.555907:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765320.555918:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765320.555939:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765320.555960:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861319040:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765320.555973:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861319168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765320.555993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765320.555997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765320.556000:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765320.556006:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1049 00000100:00080000:1.0:1775765320.556012:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765320.556017:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765320.556039:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765320.556056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861319296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765320.556095:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765320.556195:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765320/real 1775765320] req@ffffa0e9c8dece00 x1862023861319040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765320.556236:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861319040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765320.556259:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765320.556268:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765320.556277:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861319040:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765320.556320:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765320.556366:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765320.556391:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765320/real 1775765320] req@ffffa0e9c8dedf80 x1862023861319168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765320.556445:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861319168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765320.556462:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765320.556470:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765320.556476:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319168:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765320.556542:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765320.556586:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765320.556606:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765320/real 1775765320] req@ffffa0e9c8deca80 x1862023861319296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765320.556628:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861319296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765320.556653:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765320.556659:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765320.556666:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319296:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765321.131635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765321.387636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765322.155725:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765322.411551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765323.179612:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765323.435603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765324.203604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765324.459622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765325.227737:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765325.483611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765325.675680:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765325.675694:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765325.675700:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765325.675712:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1054 00000100:00080000:1.0:1775765325.675726:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765325.675735:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765325.675783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861319424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765325.675854:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765325.675861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765325.675865:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765325.675873:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1054 00000100:00080000:1.0:1775765325.675882:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765325.675891:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765325.675909:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765325.675939:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861319424:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765325.675942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861319552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765325.675967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765325.675974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765325.675976:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765325.675982:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1054 00000100:00080000:1.0:1775765325.675989:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765325.675994:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765325.676002:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765325.676021:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861319680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765325.676071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765325.676133:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765325/real 1775765325] req@ffffa0e9c8dee680 x1862023861319424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765325.676164:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861319424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765325.676179:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765325.676185:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765325.676191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861319424:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765325.676221:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765325.676249:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765325.676263:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765325/real 1775765325] req@ffffa0e9c8deea00 x1862023861319552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765325.676278:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861319552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765325.676288:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765325.676293:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765325.676297:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319552:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765325.676304:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765325.676323:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765325.676333:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765325/real 1775765325] req@ffffa0e9c8ded880 x1862023861319680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765325.676367:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861319680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765325.676377:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765325.676381:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765325.676384:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319680:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765326.251700:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765326.507629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765327.275583:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765327.531610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765328.299625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765328.555679:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765329.323665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765329.579558:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765330.347641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765330.603556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765330.795591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765330.795600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765330.795604:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765330.795624:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1059 00000100:00080000:1.0:1775765330.795655:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765330.795662:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765330.795694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861319808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765330.795716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765330.795719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765330.795720:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765330.795723:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1059 00000100:00080000:1.0:1775765330.795726:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765330.795728:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765330.795735:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765330.795744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861319936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765330.795752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765330.795755:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765330.795757:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765330.795760:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861319808:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765330.795762:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1059 00000100:00080000:1.0:1775765330.795768:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765330.795770:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765330.795773:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765330.795781:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861320064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765330.795822:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765330.795863:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765330/real 1775765330] req@ffffa0e9c8def800 x1862023861319808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765330.795890:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861319808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765330.795897:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765330.795901:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765330.795904:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861319808:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765330.795922:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765330.795938:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765330.795955:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765330/real 1775765330] req@ffffa0e9c8dedc00 x1862023861319936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765330.795966:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861319936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765330.795975:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765330.795978:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765330.795982:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861319936:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765330.795989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765330.796024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765330.796034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765330/real 1775765330] req@ffffa0e9c8def480 x1862023861320064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765330.796046:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861320064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765330.796054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765330.796057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765330.796060:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320064:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765331.371674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765331.627626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765332.395765:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765332.651705:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765333.419748:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765333.675687:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765334.443679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765334.699677:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765335.467671:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765335.723632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765335.915628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765335.915659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765335.915664:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765335.915676:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1064 00000100:00080000:1.0:1775765335.915688:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765335.915695:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765335.915746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861320192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765335.915778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765335.915782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765335.915784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765335.915789:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1064 00000100:00080000:1.0:1775765335.915792:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765335.915796:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765335.915808:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765335.915822:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861320320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765335.915835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765335.915841:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765335.915843:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861320192:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765335.915844:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765335.915849:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1064 00000100:00080000:1.0:1775765335.915853:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765335.915862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765335.915867:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765335.915881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861320448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765335.915929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765335.915983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765335/real 1775765335] req@ffffa0e9c8ded500 x1862023861320192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765335.916007:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861320192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765335.916019:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765335.916024:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765335.916029:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861320192:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765335.916055:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765335.916076:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765335.916088:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765335/real 1775765335] req@ffffa0e9c8deed80 x1862023861320320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765335.916100:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861320320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765335.916109:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765335.916113:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765335.916116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320320:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765335.916121:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765335.916136:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765335.916148:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765335/real 1775765335] req@ffffa0e9c8dec380 x1862023861320448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765335.916189:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861320448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765335.916198:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765335.916201:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765335.916204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320448:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765336.491626:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765336.747602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765337.515733:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765337.771676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765338.539639:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765338.795690:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765339.563667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765339.819572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765340.587778:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765340.843634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765341.035605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765341.035614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765341.035618:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765341.035635:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1069 00000100:00080000:0.0:1775765341.035648:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765341.035654:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765341.035690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861320576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765341.035714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765341.035717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765341.035723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765341.035727:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1069 00000100:00080000:0.0:1775765341.035732:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765341.035736:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765341.035748:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765341.035764:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861320576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765341.035765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861320704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765341.035803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765341.035806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765341.035808:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765341.035811:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1069 00000100:00080000:0.0:1775765341.035816:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765341.035819:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765341.035825:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765341.035825:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765341.035850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861320832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765341.035876:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765341/real 1775765341] req@ffffa0e9d09dd880 x1862023861320576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765341.035892:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861320576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765341.035900:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765341.035904:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765341.035907:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861320576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765341.035945:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765341.035974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765341.035984:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765341/real 1775765341] req@ffffa0e9d09dc700 x1862023861320704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765341.035993:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861320704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765341.035999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765341.036001:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765341.036004:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320704:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765341.036008:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765341.036020:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765341.036026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765341/real 1775765341] req@ffffa0e9d09df800 x1862023861320832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765341.036034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861320832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765341.036040:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765341.036042:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765341.036044:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861320832:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765341.611736:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765341.867710:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765342.635674:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765342.891694:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765343.659694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765343.915566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765344.683665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765344.939620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765345.707648:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765345.963587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765346.155562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765346.155571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765346.155576:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765346.155585:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1074 00000100:00080000:0.0:1775765346.155595:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765346.155602:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765346.155640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861320960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765346.155668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765346.155671:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765346.155673:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765346.155677:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1074 00000100:00080000:0.0:1775765346.155697:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765346.155702:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765346.155722:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765346.155736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861321088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765346.155737:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861320960:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765346.155753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765346.155756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765346.155758:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765346.155762:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1074 00000100:00080000:0.0:1775765346.155765:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765346.155768:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765346.155771:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765346.155782:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861321216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765346.155816:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765346.155887:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765346/real 1775765346] req@ffffa0e9d09dd180 x1862023861320960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765346.155908:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861320960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765346.155919:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765346.155925:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765346.155930:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861320960:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765346.155969:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765346.155990:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765346.156000:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765346/real 1775765346] req@ffffa0e9d09ddc00 x1862023861321088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765346.156009:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861321088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765346.156016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765346.156019:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765346.156022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765346.156043:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765346.156060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765346.156069:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765346/real 1775765346] req@ffffa0e9d09dd500 x1862023861321216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765346.156115:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861321216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765346.156122:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765346.156135:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765346.156137:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321216:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765346.731656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765346.987565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765347.755753:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765348.011587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765348.779607:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765349.035578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765349.803729:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765350.059638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765350.827610:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765351.083563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765351.275602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765351.275629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765351.275634:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765351.275646:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1079 00000100:00080000:1.0:1775765351.275658:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765351.275667:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765351.275717:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861321344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765351.275744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765351.275749:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765351.275751:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765351.275755:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1079 00000100:00080000:1.0:1775765351.275760:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765351.275764:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765351.275783:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765351.275798:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861321472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765351.275802:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861321344:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765351.275811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765351.275814:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765351.275832:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765351.275836:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1079 00000100:00080000:1.0:1775765351.275840:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765351.275845:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765351.275868:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765351.275881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861321600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765351.275912:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765351.275956:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765351/real 1775765351] req@ffffa0e9c8dee300 x1862023861321344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765351.275976:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861321344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765351.275984:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765351.275987:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765351.275990:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861321344:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765351.276011:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765351.276042:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765351.276052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765351/real 1775765351] req@ffffa0e9c8dec000 x1862023861321472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765351.276077:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861321472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765351.276083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765351.276086:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765351.276088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321472:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765351.276092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765351.276104:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765351.276110:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765351/real 1775765351] req@ffffa0e9c8ded180 x1862023861321600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765351.276118:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861321600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765351.276123:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765351.276125:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765351.276127:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321600:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765351.851631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765352.107629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765352.875748:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765353.131689:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765353.899698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765354.155648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765354.923619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765355.179671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765355.947691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765356.203586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765356.395637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765356.395653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765356.395662:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765356.395677:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1084 00000100:00080000:1.0:1775765356.395707:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765356.395717:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765356.395783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861321728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765356.395829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765356.395834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765356.395837:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765356.395842:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1084 00000100:00080000:1.0:1775765356.395846:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765356.395851:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765356.395865:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765356.395883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861321856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765356.395898:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765356.395903:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765356.395905:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765356.395911:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1084 00000100:00100000:2.0:1775765356.395921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861321728:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765356.395944:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765356.395953:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765356.395962:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765356.395980:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861321984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765356.396034:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765356.396099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765356/real 1775765356] req@ffffa0e9c8ded180 x1862023861321728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765356.396125:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861321728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765356.396140:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765356.396146:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765356.396151:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861321728:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765356.396161:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765356.396189:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765356.396203:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765356/real 1775765356] req@ffffa0e9c8dec000 x1862023861321856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765356.396227:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861321856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765356.396237:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765356.396240:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765356.396244:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321856:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765356.396278:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765356.396298:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765356.396310:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765356/real 1775765356] req@ffffa0e9c8dee300 x1862023861321984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765356.396339:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861321984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765356.396350:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765356.396353:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765356.396357:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861321984:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765356.971714:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765357.227627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765357.995604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765358.251594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765359.019644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765359.275594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765360.043654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765360.299586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765361.067583:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765361.323600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765361.515575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765361.515583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765361.515587:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765361.515595:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1090 00000100:00080000:0.0:1775765361.515605:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765361.515611:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765361.515662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861322112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765361.515687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765361.515689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765361.515691:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765361.515693:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1090 00000100:00080000:0.0:1775765361.515696:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765361.515699:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765361.515707:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765361.515714:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861322240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765361.515721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765361.515724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765361.515725:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765361.515727:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1090 00000100:00080000:0.0:1775765361.515729:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765361.515731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765361.515733:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765361.515739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861322368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765361.515780:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861322112:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765361.515868:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322240:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765361.515886:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322368:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765362.091770:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765362.347612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765363.115612:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765363.371551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765364.139624:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765364.395565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775765364.587805:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775765364.587872:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765364.587891:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765364.587936:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765364.587946:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765364.587996:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765361/real 1775765364] req@ffffa0e9d09de680 x1862023861322112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765364.588041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861322112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765364.588053:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765364.588059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765364.588064:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861322112:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765364.588072:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765361/real 1775765364] req@ffffa0e9d09dca80 x1862023861322240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765364.588084:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861322240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765364.588092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765364.588096:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765364.588100:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322240:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765364.588107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765361/real 1775765364] req@ffffa0e9d09ddf80 x1862023861322368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765364.588121:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861322368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765364.588130:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765364.588133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765364.588136:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322368:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765365.163635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765365.419608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765366.187620:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765366.443541:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765366.635614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765366.635623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765366.635626:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765366.635634:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1095 00000100:00080000:1.0:1775765366.635644:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765366.635649:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765366.635681:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861322496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765366.635704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765366.635707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765366.635708:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765366.635711:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1095 00000100:00080000:1.0:1775765366.635714:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765366.635716:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765366.635724:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765366.635733:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861322624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765366.635742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765366.635749:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861322496:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765366.635758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765366.635760:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765366.635764:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1095 00000100:00080000:1.0:1775765366.635767:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765366.635770:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765366.635775:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765366.635784:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861322752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765366.635825:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765366.635868:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765366/real 1775765366] req@ffffa0e9c8dec380 x1862023861322496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765366.635891:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861322496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765366.635903:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765366.635907:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765366.635912:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861322496:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765366.635938:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765366.635973:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765366.635986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765366/real 1775765366] req@ffffa0e9c8deed80 x1862023861322624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765366.635998:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861322624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765366.636006:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765366.636010:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765366.636014:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322624:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765366.636021:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765366.636051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765366.636064:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765366/real 1775765366] req@ffffa0e9c8ded500 x1862023861322752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765366.636095:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861322752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765366.636104:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765366.636108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765366.636111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861322752:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765367.211612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765367.467617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765368.235763:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765368.491678:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765369.259669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765369.515563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765370.283603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765370.539564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765371.307702:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765371.563562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765371.755613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765371.755621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765371.755626:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765371.755634:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1100 00000100:00080000:0.0:1775765371.755644:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765371.755650:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765371.755692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861322880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765371.755717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765371.755719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765371.755721:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765371.755724:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1100 00000100:00080000:0.0:1775765371.755726:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765371.755728:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765371.755736:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765371.755745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861323008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765371.755759:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861322880:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765371.755775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765371.755777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765371.755779:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765371.755783:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1100 00000100:00080000:0.0:1775765371.755786:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765371.755788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765371.755792:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765371.755802:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861323136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765371.755830:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765371.755871:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765371/real 1775765371] req@ffffa0e9d09ddf80 x1862023861322880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765371.755887:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861322880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765371.755895:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765371.755898:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765371.755902:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861322880:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765371.755922:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765371.755937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765371.755948:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765371/real 1775765371] req@ffffa0e9d09dca80 x1862023861323008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765371.755961:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861323008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765371.755985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765371.755989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765371.755993:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323008:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765371.756000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765371.756019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765371.756027:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765371/real 1775765371] req@ffffa0e9d09de680 x1862023861323136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765371.756035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861323136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765371.756041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765371.756043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765371.756045:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323136:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765372.331654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765372.587570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765373.355641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765373.611602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765374.379587:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765374.635551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765375.403636:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765375.659584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765376.427558:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765376.683539:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765376.875588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765376.875599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765376.875605:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765376.875614:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1105 00000100:00080000:3.0:1775765376.875625:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765376.875630:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765376.875662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861323264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765376.875692:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765376.875695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765376.875696:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765376.875699:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1105 00000100:00080000:3.0:1775765376.875701:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765376.875704:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765376.875712:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765376.875721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861323392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765376.875732:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861323264:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765376.875734:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765376.875741:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765376.875743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765376.875748:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1105 00000100:00080000:3.0:1775765376.875753:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765376.875757:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765376.875763:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765376.875790:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861323520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765376.875797:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765376.875843:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765376/real 1775765376] req@ffffa0e9d0540380 x1862023861323264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765376.875877:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861323264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765376.875885:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765376.875888:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765376.875891:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861323264:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765376.875910:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765376.875927:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765376.875936:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765376/real 1775765376] req@ffffa0e9d0542300 x1862023861323392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765376.875944:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861323392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765376.875949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765376.875951:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765376.875953:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323392:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765376.875956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765376.875966:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765376.875971:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765376/real 1775765376] req@ffffa0e9d0543b80 x1862023861323520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765376.875979:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861323520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765376.875992:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765376.875994:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765376.875996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323520:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765377.451708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765377.707584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765378.475712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765378.731592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765379.499696:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765379.755567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765380.523707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765380.779582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765381.547900:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765381.803619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765381.995668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765381.995683:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765381.995701:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765381.995715:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1110 00000100:00080000:0.0:1775765381.995732:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765381.995741:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765381.995806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861323648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765381.995848:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765381.995852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765381.995878:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765381.995884:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1110 00000100:00080000:0.0:1775765381.995890:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765381.995896:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765381.995931:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765381.995949:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861323776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765381.995963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861323648:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765381.995969:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765381.995992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765381.995994:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765381.996001:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1110 00000100:00080000:0.0:1775765381.996007:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765381.996012:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765381.996019:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765381.996041:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861323904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765381.996096:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765381.996193:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765381/real 1775765381] req@ffffa0e9d09de680 x1862023861323648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765381.996219:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861323648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765381.996233:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765381.996238:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765381.996243:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861323648:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765381.996273:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765381.996302:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765381.996316:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765381/real 1775765381] req@ffffa0e9d09dca80 x1862023861323776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765381.996366:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861323776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765381.996376:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765381.996380:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765381.996384:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323776:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765381.996391:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765381.996430:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765381.996448:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765381/real 1775765381] req@ffffa0e9d09ddf80 x1862023861323904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765381.996462:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861323904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765381.996471:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765381.996475:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765381.996479:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861323904:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765382.571660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765382.827616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765383.595797:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765383.851630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765384.619678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765384.875622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765385.643667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765385.899605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765386.667721:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765386.923637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765387.115716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765387.115732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765387.115740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765387.115755:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1115 00000100:00080000:0.0:1775765387.115773:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765387.115783:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765387.115857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861324032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765387.115928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765387.115933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765387.115935:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765387.115941:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1115 00000100:00080000:0.0:1775765387.115946:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765387.115952:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765387.115966:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765387.115988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861324160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765387.116012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765387.116022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765387.116027:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861324032:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765387.116028:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765387.116065:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1115 00000100:00080000:0.0:1775765387.116075:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765387.116121:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765387.116132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765387.116151:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861324288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765387.116174:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765387.116231:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765387/real 1775765387] req@ffffa0e9d09dd500 x1862023861324032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765387.116250:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861324032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765387.116261:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765387.116266:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765387.116270:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861324032:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765387.116289:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765387.116312:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765387.116321:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765387/real 1775765387] req@ffffa0e9d09ddc00 x1862023861324160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765387.116329:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861324160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765387.116334:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765387.116337:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765387.116339:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324160:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765387.116343:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765387.116353:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765387.116359:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765387/real 1775765387] req@ffffa0e9d09dd180 x1862023861324288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765387.116377:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861324288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765387.116383:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765387.116385:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765387.116387:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324288:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765387.691651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765387.947600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765388.715643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765388.971615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765389.739740:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765389.995605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765390.763694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765391.019641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765391.787671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765392.043624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765392.235601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765392.235609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765392.235613:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765392.235621:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1120 00000100:00080000:3.0:1775765392.235631:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765392.235636:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765392.235668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861324416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765392.235691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765392.235694:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765392.235695:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765392.235698:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1120 00000100:00080000:3.0:1775765392.235700:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765392.235702:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765392.235709:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765392.235717:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861324544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765392.235724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765392.235757:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861324416:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765392.235759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765392.235766:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765392.235775:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1120 00000100:00080000:3.0:1775765392.235781:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765392.235785:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765392.235794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765392.235813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861324672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765392.235852:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765392.235916:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765392/real 1775765392] req@ffffa0e9d0540000 x1862023861324416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765392.235942:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861324416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765392.235959:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765392.235965:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765392.235972:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861324416:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765392.235982:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765392.236008:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765392.236078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765392/real 1775765392] req@ffffa0e9d0542a00 x1862023861324544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765392.236086:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861324544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765392.236092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765392.236094:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765392.236097:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324544:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765392.236125:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765392.236138:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765392.236144:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765392/real 1775765392] req@ffffa0e9d0541500 x1862023861324672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765392.236152:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861324672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765392.236156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765392.236158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765392.236160:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324672:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765392.811627:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765393.067658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765393.835607:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765394.091577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765394.859611:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765395.115573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765395.883625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765396.139581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765396.907652:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765397.163593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765397.355599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765397.355610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765397.355615:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765397.355625:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1125 00000100:00080000:1.0:1775765397.355637:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765397.355645:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765397.355682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861324800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765397.355704:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765397.355707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765397.355708:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765397.355711:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1125 00000100:00080000:1.0:1775765397.355714:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765397.355717:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765397.355724:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765397.355733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861324928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765397.355754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765397.355757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765397.355759:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765397.355766:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1125 00000100:00100000:2.0:1775765397.355768:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861324800:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765397.355770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765397.355774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765397.355779:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765397.355795:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861325056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765397.355859:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765397.355905:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765397/real 1775765397] req@ffffa0e9c8def480 x1862023861324800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765397.355926:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861324800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765397.355937:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765397.355941:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765397.355946:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861324800:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765397.355954:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765397.356005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765397.356026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765397/real 1775765397] req@ffffa0e9c8dedc00 x1862023861324928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765397.356040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861324928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765397.356049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765397.356053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765397.356057:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861324928:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765397.356081:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765397.356103:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765397.356115:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765397/real 1775765397] req@ffffa0e9c8def800 x1862023861325056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765397.356143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861325056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765397.356152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765397.356155:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765397.356158:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325056:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765397.931592:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765398.187560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765398.955619:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765399.211588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765399.979735:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765400.235618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765401.003580:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765401.259579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765402.027655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765402.283589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765402.475636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765402.475644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765402.475649:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765402.475656:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1130 00000100:00080000:3.0:1775765402.475665:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765402.475671:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765402.475705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861325184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765402.475731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765402.475734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765402.475735:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765402.475753:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1130 00000100:00080000:3.0:1775765402.475756:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765402.475779:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765402.475806:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765402.475825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861325312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765402.475836:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861325184:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765402.475856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765402.475862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765402.475865:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765402.475872:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1130 00000100:00080000:3.0:1775765402.475894:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765402.475899:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765402.475906:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765402.475921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861325440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765402.475966:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765402.476024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765402/real 1775765402] req@ffffa0e9d0542680 x1862023861325184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765402.476041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861325184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765402.476050:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765402.476054:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765402.476057:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861325184:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765402.476076:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765402.476112:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765402.476121:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765402/real 1775765402] req@ffffa0e9d0541880 x1862023861325312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765402.476130:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861325312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765402.476135:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765402.476137:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765402.476139:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325312:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765402.476143:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765402.476152:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765402.476157:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765402/real 1775765402] req@ffffa0e9d0541500 x1862023861325440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765402.476164:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861325440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765402.476169:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765402.476171:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765402.476173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325440:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765403.051707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765403.307599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765404.075628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765404.331585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765405.099634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765405.355573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765406.123688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765406.379697:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765407.147580:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765407.403548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765407.595583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765407.595593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765407.595599:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765407.595608:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1136 00000100:00080000:3.0:1775765407.595619:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765407.595626:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765407.595662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861325568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765407.595691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765407.595695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765407.595713:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765407.595718:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1136 00000100:00080000:3.0:1775765407.595724:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765407.595729:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765407.595740:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765407.595748:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861325568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775765407.595756:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861325696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765407.595770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765407.595774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765407.595776:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765407.595780:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1136 00000100:00080000:3.0:1775765407.595784:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765407.595788:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765407.595794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765407.595806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861325824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765407.595826:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765407.595899:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765407/real 1775765407] req@ffffa0e9d0542a00 x1862023861325568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765407.595920:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861325568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765407.595929:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765407.595932:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765407.595937:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861325568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765407.595961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765407.595988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765407.596002:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765407/real 1775765407] req@ffffa0e9d0540000 x1862023861325696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765407.596016:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861325696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765407.596024:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765407.596028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765407.596031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325696:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765407.596038:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765407.596056:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765407.596065:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765407/real 1775765407] req@ffffa0e9d0543b80 x1862023861325824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765407.596093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861325824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765407.596102:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765407.596106:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765407.596109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861325824:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765408.171716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765408.427618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765409.195681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765409.451600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765410.219615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765410.475575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765411.243635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765411.499548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765412.267691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765412.523565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765412.715558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765412.715566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765412.715571:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765412.715579:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1141 00000100:00080000:3.0:1775765412.715588:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765412.715593:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765412.715629:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861325952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765412.715677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765412.715682:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765412.715684:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765412.715689:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1141 00000100:00080000:3.0:1775765412.715692:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765412.715696:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765412.715704:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765412.715716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861326080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765412.715725:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861325952:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765412.715725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765412.715731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765412.715734:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765412.715737:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1141 00000100:00080000:3.0:1775765412.715740:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765412.715742:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765412.715745:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765412.715753:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861326208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765412.715796:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765412.715870:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765412/real 1775765412] req@ffffa0e9d0542300 x1862023861325952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765412.715892:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861325952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765412.715903:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765412.715906:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765412.715909:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861325952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765412.715928:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765412.715958:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765412.715971:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765412/real 1775765412] req@ffffa0e9d0540380 x1862023861326080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765412.715985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861326080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765412.715993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765412.715997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765412.716001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326080:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765412.716008:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765412.716026:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765412.716034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765412/real 1775765412] req@ffffa0e9d0541c00 x1862023861326208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765412.716042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861326208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765412.716048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765412.716050:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765412.716052:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326208:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765413.291586:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765413.547576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765414.315625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765414.571603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765415.339658:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765415.595548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765416.363606:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765416.619586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765417.387607:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765417.643576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765417.835551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765417.835559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765417.835562:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765417.835587:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1146 00000100:00080000:3.0:1775765417.835596:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765417.835601:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765417.835632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861326336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765417.835656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765417.835659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765417.835660:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765417.835663:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1146 00000100:00080000:3.0:1775765417.835665:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765417.835668:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765417.835676:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765417.835689:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861326464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765417.835702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765417.835703:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861326336:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765417.835708:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765417.835714:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765417.835719:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1146 00000100:00080000:3.0:1775765417.835723:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765417.835727:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765417.835733:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765417.835754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861326592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765417.835773:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765417.835833:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765417/real 1775765417] req@ffffa0e9d0543800 x1862023861326336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765417.835849:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861326336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765417.835858:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765417.835861:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765417.835864:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861326336:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765417.835882:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765417.835900:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765417.835908:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765417/real 1775765417] req@ffffa0e9d0540700 x1862023861326464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765417.835916:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861326464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765417.835921:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765417.835924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765417.835926:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326464:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765417.835930:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765417.835940:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765417.835945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765417/real 1775765417] req@ffffa0e9d0540e00 x1862023861326592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765417.835966:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861326592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765417.835974:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765417.835978:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765417.835981:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326592:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765418.411586:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765418.667597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765419.435586:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765419.691598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765420.459600:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765420.715559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765421.483622:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765421.739565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765422.507571:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765422.763553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765422.955594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765422.955603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765422.955607:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765422.955615:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1151 00000100:00080000:3.0:1775765422.955625:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765422.955630:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765422.955663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861326720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765422.955687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765422.955690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765422.955691:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765422.955694:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1151 00000100:00080000:3.0:1775765422.955696:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765422.955698:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765422.955705:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765422.955713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861326848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765422.955721:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765422.955724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765422.955725:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765422.955728:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1151 00000100:00080000:3.0:1775765422.955763:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765422.955769:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765422.955770:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861326720:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775765422.955774:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765422.955785:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861326976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765422.955835:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765422.955882:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765422/real 1775765422] req@ffffa0e9d0540a80 x1862023861326720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765422.955903:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861326720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765422.955915:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765422.955920:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765422.955924:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861326720:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765422.955932:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765422.955954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765422.955974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765422/real 1775765422] req@ffffa0e9d0542d80 x1862023861326848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765422.955987:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861326848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765422.955997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765422.956000:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765422.956004:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326848:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765422.956028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765422.956047:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765422.956057:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765422/real 1775765422] req@ffffa0e9d0541f80 x1862023861326976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765422.956070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861326976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765422.956079:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765422.956082:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765422.956086:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861326976:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765423.531613:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765423.787546:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765424.555625:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765424.811602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765425.579604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765425.835564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765426.603637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765426.859560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765427.627621:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765427.883566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765428.075597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765428.075608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765428.075613:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765428.075622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1156 00000100:00080000:1.0:1775765428.075631:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765428.075637:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765428.075740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861327104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765428.075761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765428.075764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765428.075765:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765428.075768:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1156 00000100:00080000:1.0:1775765428.075771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765428.075773:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765428.075795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765428.075810:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861327104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765428.075814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861327232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765428.075830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765428.075832:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765428.075834:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765428.075837:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1156 00000100:00080000:1.0:1775765428.075840:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765428.075842:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765428.075846:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765428.075856:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861327360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765428.075919:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765428.075946:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327360:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765428.651644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765428.907594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765429.675750:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765429.931635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765430.699872:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765430.955596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765431.147759:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765431.147802:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765431.147837:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765431.147895:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765431.147906:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765431.147997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765428/real 1775765431] req@ffffa0e9c8deea00 x1862023861327232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765431.148027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861327232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765431.148041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765431.148046:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765431.148051:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765431.148060:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765428/real 1775765431] req@ffffa0e9c8dee680 x1862023861327360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765431.148075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861327360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765431.148083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765431.148087:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765431.148091:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327360:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765431.148099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765428/real 1775765431] req@ffffa0e9c8ded880 x1862023861327104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765431.148142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861327104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765431.148152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765431.148156:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765431.148160:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861327104:192.168.202.104@tcp:250:kworker.0 00010000:00010000:3.0:1775765431.723673:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765431.979648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765432.747739:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765433.003706:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765433.195765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765433.195777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765433.195784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765433.195797:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1161 00000100:00080000:3.0:1775765433.195810:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765433.195829:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765433.195878:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861327488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765433.195911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765433.195915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765433.195917:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765433.195920:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1161 00000100:00080000:3.0:1775765433.195946:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765433.195950:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765433.195965:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765433.195983:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861327616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765433.195990:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861327488:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765433.196006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765433.196010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765433.196014:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765433.196021:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1161 00000100:00080000:3.0:1775765433.196029:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765433.196034:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765433.196044:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765433.196062:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861327744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765433.196106:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765433.196172:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765433/real 1775765433] req@ffffa0e9d0541180 x1862023861327488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765433.196199:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861327488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765433.196240:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765433.196247:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765433.196253:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861327488:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765433.196298:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765433.196334:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765433.196351:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765433/real 1775765433] req@ffffa0e9d0543480 x1862023861327616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765433.196369:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861327616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765433.196394:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765433.196399:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765433.196404:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327616:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765433.196413:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765433.196445:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765433.196461:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765433/real 1775765433] req@ffffa0e9d0543100 x1862023861327744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765433.196477:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861327744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765433.196529:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765433.196534:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765433.196538:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861327744:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765433.771838:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765434.027654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765434.795616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765435.051611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765435.819637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765436.075575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765436.843683:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765437.099645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765437.867697:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765438.123630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765438.315627:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765438.315640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765438.315646:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765438.315658:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1166 00000100:00080000:3.0:1775765438.315672:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765438.315681:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765438.315731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861327872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765438.315815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765438.315821:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765438.315823:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765438.315830:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1166 00000100:00080000:3.0:1775765438.315838:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765438.315845:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765438.315859:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765438.315869:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861327872:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775765438.315886:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861328000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765438.315904:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765438.315909:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765438.315912:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765438.315918:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1166 00000100:00080000:3.0:1775765438.315934:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765438.315941:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:2.0:1775765438.315947:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:3.0:1775765438.315953:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765438.315977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861328128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765438.316012:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765438/real 1775765438] req@ffffa0e9d0543100 x1862023861327872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765438.316050:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861327872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765438.316062:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765438.316067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765438.316072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861327872:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765438.316100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765438.316124:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765438.316136:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765438/real 1775765438] req@ffffa0e9d0543480 x1862023861328000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765438.316147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861328000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765438.316155:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765438.316158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765438.316161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328000:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765438.316166:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765438.316181:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765438.316188:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765438/real 1775765438] req@ffffa0e9d0541180 x1862023861328128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765438.316198:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861328128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765438.316218:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765438.316221:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765438.316225:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328128:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765438.891670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765439.147653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765439.915674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765440.171639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765440.939652:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765441.195602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765441.963661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765442.219571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765442.987736:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765443.243594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765443.435727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765443.435736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765443.435740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765443.435748:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1171 00000100:00080000:1.0:1775765443.435758:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765443.435764:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765443.435840:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861328256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765443.435879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765443.435884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765443.435886:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765443.435891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1171 00000100:00080000:1.0:1775765443.435916:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765443.435922:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765443.435935:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765443.435962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861328384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765443.435979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765443.435986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765443.435987:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861328256:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765443.435989:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765443.436002:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1171 00000100:00080000:1.0:1775765443.436006:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765443.436010:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765443.436017:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765443.436040:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861328512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765443.436083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765443.436149:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765443/real 1775765443] req@ffffa0e9c8deca80 x1862023861328256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765443.436175:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861328256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765443.436190:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765443.436212:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765443.436215:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861328256:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765443.436234:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765443.436250:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765443.436283:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765443/real 1775765443] req@ffffa0e9c8dedf80 x1862023861328384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765443.436318:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861328384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765443.436332:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765443.436336:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765443.436340:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328384:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765443.436346:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765443.436368:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765443.436378:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765443/real 1775765443] req@ffffa0e9c8dece00 x1862023861328512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765443.436404:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861328512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765443.436409:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765443.436411:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765443.436413:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328512:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765444.011724:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765444.267708:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765445.035752:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765445.291696:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765446.059704:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765446.315633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765447.083653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765447.339608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765448.107640:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765448.363604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765448.555606:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765448.555630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765448.555634:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765448.555642:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1177 00000100:00080000:2.0:1775765448.555651:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765448.555657:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765448.555686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861328640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765448.555712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765448.555715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765448.555716:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765448.555719:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1177 00000100:00080000:2.0:1775765448.555721:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765448.555723:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765448.555731:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765448.555739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861328768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765448.555745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765448.555748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765448.555749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765448.555752:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1177 00000100:00080000:2.0:1775765448.555754:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765448.555756:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765448.555759:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765448.555765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861328896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765448.555775:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861328640:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775765448.555840:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765448.555879:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765448/real 1775765448] req@ffffa0e9d0713480 x1862023861328640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765448.555895:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861328640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765448.555903:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765448.555907:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765448.555910:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861328640:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765448.555915:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765448.555932:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765448.555944:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765448/real 1775765448] req@ffffa0e9d0711500 x1862023861328768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765448.555962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861328768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765448.555968:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765448.555970:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765448.555972:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328768:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765448.555992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765448.556005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765448.556011:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765448/real 1775765448] req@ffffa0e9d0712300 x1862023861328896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765448.556019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861328896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765448.556025:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765448.556027:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765448.556029:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861328896:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765449.131750:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765449.387639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765450.155683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765450.411618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765451.179747:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765451.435606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765452.203630:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765452.459555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765453.227740:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765453.483592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765453.675696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765453.675706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765453.675711:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765453.675721:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1182 00000100:00080000:1.0:1775765453.675733:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765453.675741:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765453.675782:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861329024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765453.675814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765453.675835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765453.675837:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765453.675841:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1182 00000100:00080000:1.0:1775765453.675845:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765453.675850:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765453.675866:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765453.675879:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861329152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765453.675891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765453.675895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765453.675897:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765453.675901:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1182 00000100:00080000:1.0:1775765453.675905:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765453.675909:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765453.675913:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765453.675923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861329280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765453.675931:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861329024:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775765453.676001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765453.676101:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765453/real 1775765453] req@ffffa0e9c8def100 x1862023861329024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765453.676143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861329024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765453.676164:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765453.676169:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765453.676174:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861329024:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765453.676181:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765453.676226:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765453.676240:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765453/real 1775765453] req@ffffa0e9c8dec700 x1862023861329152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765453.676253:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861329152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765453.676262:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765453.676266:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765453.676269:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329152:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765453.676295:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765453.676328:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765453.676363:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765453/real 1775765453] req@ffffa0e9c8defb80 x1862023861329280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765453.676383:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861329280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765453.676396:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765453.676401:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765453.676406:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329280:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765454.251654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765454.507618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765455.275682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765455.531586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765456.299575:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765456.555555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765457.323618:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765457.579565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765458.347701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765458.603592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765458.795616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765458.795631:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765458.795638:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765458.795651:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1187 00000100:00080000:1.0:1775765458.795666:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765458.795676:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765458.795757:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861329408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765458.795820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765458.795826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765458.795830:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765458.795837:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1187 00000100:00080000:1.0:1775765458.795844:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765458.795853:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765458.795877:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765458.795888:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861329408:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765458.795899:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861329536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765458.795915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765458.795919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765458.795922:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765458.795926:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1187 00000100:00080000:1.0:1775765458.795931:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765458.795935:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765458.795940:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765458.795953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861329664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765458.795967:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765458.796024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765458/real 1775765458] req@ffffa0e9c8defb80 x1862023861329408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765458.796051:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861329408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765458.796063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765458.796068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765458.796073:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861329408:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765458.796098:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765458.796120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765458.796133:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765458/real 1775765458] req@ffffa0e9c8dec700 x1862023861329536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765458.796145:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861329536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765458.796154:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765458.796158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765458.796161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329536:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765458.796167:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765458.796204:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765458.796215:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765458/real 1775765458] req@ffffa0e9c8def100 x1862023861329664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765458.796228:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861329664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765458.796236:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765458.796239:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765458.796243:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329664:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765459.371572:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765459.627569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765460.395619:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765460.651582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765461.419629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765461.675619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765462.443588:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765462.699608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765463.467597:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765463.723588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765463.915547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765463.915554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765463.915559:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765463.915566:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1192 00000100:00080000:1.0:1775765463.915611:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765463.915617:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765463.915649:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861329792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765463.915673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765463.915675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765463.915677:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765463.915680:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1192 00000100:00080000:1.0:1775765463.915682:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765463.915684:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765463.915693:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765463.915702:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861329920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765463.915709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765463.915712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765463.915714:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765463.915716:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1192 00000100:00080000:1.0:1775765463.915718:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765463.915721:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765463.915727:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861329792:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775765463.915727:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765463.915735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861330048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765463.915801:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765463.915845:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765463/real 1775765463] req@ffffa0e9c8dece00 x1862023861329792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765463.915861:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861329792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765463.915868:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765463.915872:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765463.915875:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861329792:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765463.915880:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765463.915899:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765463.915915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765463/real 1775765463] req@ffffa0e9c8dedf80 x1862023861329920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765463.915936:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861329920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765463.915942:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765463.915944:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765463.915947:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861329920:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765463.915971:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765463.915986:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765463.915993:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765463/real 1775765463] req@ffffa0e9c8deca80 x1862023861330048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765463.916012:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861330048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765463.916017:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765463.916019:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765463.916022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330048:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765464.491628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765464.747599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765465.515607:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765465.771610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765466.539655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765466.795606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765467.563630:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765467.819568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765468.587634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765468.843611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765469.035671:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765469.035684:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765469.035690:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765469.035703:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1197 00000100:00080000:3.0:1775765469.035717:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765469.035726:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765469.035790:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861330176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765469.035824:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765469.035829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765469.035832:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765469.035837:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1197 00000100:00080000:3.0:1775765469.035841:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765469.035845:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765469.035857:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765469.035878:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861330304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765469.035902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765469.035912:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861330176:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765469.035912:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765469.035918:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765469.035956:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1197 00000100:00080000:3.0:1775765469.035966:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765469.035975:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765469.035989:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765469.036010:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861330432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765469.036036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765469.036094:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765469/real 1775765469] req@ffffa0e9d0541f80 x1862023861330176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765469.036120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861330176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765469.036146:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765469.036152:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765469.036157:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861330176:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765469.036188:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765469.036221:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765469.036236:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765469/real 1775765469] req@ffffa0e9d0542d80 x1862023861330304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765469.036252:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861330304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765469.036262:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765469.036284:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765469.036288:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330304:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765469.036321:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765469.036349:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765469.036364:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765469/real 1775765469] req@ffffa0e9d0540a80 x1862023861330432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765469.036379:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861330432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765469.036389:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765469.036393:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765469.036397:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330432:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765469.611682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765469.867599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765470.635699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765470.891629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765471.659585:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765471.915632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765472.683779:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765472.939621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765473.707711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765473.963615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765474.155630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765474.155642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765474.155648:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765474.155660:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1202 00000100:00080000:3.0:1775765474.155674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765474.155682:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765474.155762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861330560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765474.155802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765474.155807:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765474.155809:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765474.155814:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1202 00000100:00080000:3.0:1775765474.155818:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765474.155822:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765474.155834:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765474.155848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861330688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765474.155888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765474.155893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765474.155896:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765474.155902:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1202 00000100:00080000:3.0:1775765474.155907:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765474.155912:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765474.155919:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765474.155934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861330816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765474.155943:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861330560:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775765474.156028:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765474.156094:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765474/real 1775765474] req@ffffa0e9d0540e00 x1862023861330560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765474.156119:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861330560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765474.156133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765474.156139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765474.156145:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861330560:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765474.156157:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765474.156183:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765474.156198:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765474/real 1775765474] req@ffffa0e9d0540700 x1862023861330688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765474.156212:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861330688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765474.156223:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765474.156227:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765474.156231:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330688:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765474.156294:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765474.156356:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765474.156378:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765474/real 1775765474] req@ffffa0e9d0543800 x1862023861330816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765474.156403:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861330816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765474.156433:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765474.156438:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765474.156442:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861330816:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765474.731617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765474.987587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765475.755671:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765476.011601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765476.779742:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765477.035624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765477.803684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765478.059569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765478.827631:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765479.083581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765479.275608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765479.275631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765479.275635:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765479.275644:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1207 00000100:00080000:1.0:1775765479.275653:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765479.275659:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765479.275690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861330944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765479.275714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765479.275716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765479.275718:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765479.275720:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1207 00000100:00080000:1.0:1775765479.275723:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765479.275725:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765479.275733:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765479.275741:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861331072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765479.275749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765479.275752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765479.275753:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765479.275758:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1207 00000100:00100000:0.0:1775765479.275759:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861330944:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765479.275760:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765479.275765:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765479.275769:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765479.275777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861331200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765479.275820:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765479.275862:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765479/real 1775765479] req@ffffa0e9c8ded880 x1862023861330944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765479.275881:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861330944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765479.275890:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765479.275892:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765479.275895:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861330944:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765479.275913:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765479.275931:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765479.275940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765479/real 1775765479] req@ffffa0e9c8dee680 x1862023861331072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765479.275959:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861331072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765479.275965:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765479.275968:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765479.275970:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331072:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765479.275974:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765479.275988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765479.275998:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765479/real 1775765479] req@ffffa0e9c8deea00 x1862023861331200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765479.276010:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861331200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765479.276018:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765479.276022:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765479.276026:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331200:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765479.851693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765480.107595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765480.875635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765481.131602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765481.899749:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765482.155578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765482.923726:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765483.179601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765483.947625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765484.203644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765484.395647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765484.395661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765484.395668:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765484.395682:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1212 00000100:00080000:2.0:1775765484.395698:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765484.395712:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765484.395763:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861331328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765484.395802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765484.395807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765484.395809:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765484.395814:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1212 00000100:00080000:2.0:1775765484.395819:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765484.395848:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765484.395864:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765484.395874:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861331328:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765484.395890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861331456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765484.395909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765484.395913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765484.395917:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765484.395923:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1212 00000100:00080000:2.0:1775765484.395929:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765484.395934:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765484.395940:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765484.395960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861331584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765484.395999:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765484.396061:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765484/real 1775765484] req@ffffa0e9d0713100 x1862023861331328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765484.396085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861331328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765484.396117:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765484.396123:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765484.396128:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861331328:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765484.396193:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765484.396246:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765484.396263:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765484/real 1775765484] req@ffffa0e9d0711880 x1862023861331456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765484.396280:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861331456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765484.396290:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765484.396295:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765484.396299:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331456:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765484.396321:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765484.396346:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765484.396358:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765484/real 1775765484] req@ffffa0e9d0712300 x1862023861331584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765484.396372:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861331584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765484.396382:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765484.396387:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765484.396390:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331584:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765484.971615:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765485.227585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765485.995648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765486.251608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765487.019640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765487.275687:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765488.043739:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765488.299635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765489.067654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765489.323656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765489.515649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765489.515665:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765489.515672:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765489.515684:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1218 00000100:00080000:1.0:1775765489.515697:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765489.515706:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765489.515760:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861331712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765489.515820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765489.515828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765489.515831:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765489.515839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1218 00000100:00080000:1.0:1775765489.515848:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765489.515857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765489.515871:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765489.515894:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861331712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765489.515896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861331840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765489.515929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765489.515936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765489.515940:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765489.515949:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1218 00000100:00080000:1.0:1775765489.515981:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765489.515989:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:1.0:1775765489.515996:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765489.516040:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765489.516050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765489/real 1775765489] req@ffffa0e9c8def800 x1862023861331712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765489.516067:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861331968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765489.516085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861331712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765489.516125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765489.516136:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765489.516144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861331712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765489.516183:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765489.516224:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765489.516255:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765489/real 1775765489] req@ffffa0e9c8dedc00 x1862023861331840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765489.516281:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861331840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765489.516298:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765489.516306:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765489.516313:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331840:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765489.516325:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765489.516379:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765489.516394:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765489/real 1775765489] req@ffffa0e9c8def480 x1862023861331968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765489.516409:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861331968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765489.516419:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765489.516423:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765489.516427:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861331968:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765490.091650:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765490.347593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765491.115700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765491.371650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765492.139661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765492.395626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765493.163606:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765493.419589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765494.187722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765494.443609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765494.635585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765494.635595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765494.635600:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765494.635609:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1223 00000100:00080000:3.0:1775765494.635620:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765494.635627:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765494.635665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861332096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765494.635695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765494.635718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765494.635721:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765494.635728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1223 00000100:00080000:3.0:1775765494.635736:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765494.635742:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765494.635749:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861332096:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775765494.635758:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765494.635808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861332224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765494.635833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765494.635840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765494.635844:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765494.635853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1223 00000100:00080000:3.0:1775765494.635862:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765494.635870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765494.635882:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765494.635911:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861332352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765494.635937:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765494.636030:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332352:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765495.211649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765495.467560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765496.235709:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765496.491674:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765497.259674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765497.515626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775765497.707749:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775765497.707790:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765497.707806:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765497.707846:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765497.707854:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765497.707973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765494/real 1775765497] req@ffffa0e9d0542300 x1862023861332352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765497.708019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861332352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765497.708081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765497.708089:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765497.708096:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332352:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765497.708108:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765494/real 1775765497] req@ffffa0e9d0540380 x1862023861332224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765497.708148:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861332224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765497.708170:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765497.708176:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765497.708181:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765497.708191:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765494/real 1775765497] req@ffffa0e9d0541c00 x1862023861332096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765497.708209:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861332096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765497.708220:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765497.708225:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765497.708230:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861332096:192.168.202.104@tcp:250:kworker.0 00010000:00010000:0.0:1775765498.283685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765498.539592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765499.307630:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765499.563660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765499.755636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765499.755654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765499.755662:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765499.755680:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1228 00000100:00080000:3.0:1775765499.755698:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765499.755734:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765499.755786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861332480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765499.755843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765499.755851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765499.755854:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765499.755862:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1228 00000100:00080000:3.0:1775765499.755872:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765499.755880:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765499.755896:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765499.755924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861332608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765499.755947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765499.755954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775765499.755966:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861332480:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765499.755979:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765499.755992:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1228 00000100:00080000:3.0:1775765499.756029:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765499.756040:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765499.756052:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765499.756077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861332736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765499.756097:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765499.756228:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765499/real 1775765499] req@ffffa0e9d0543b80 x1862023861332480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765499.756274:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861332480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765499.756295:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765499.756304:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765499.756328:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861332480:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765499.756355:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765499.756405:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765499.756431:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765499/real 1775765499] req@ffffa0e9d0540000 x1862023861332608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765499.756456:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861332608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765499.756471:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765499.756555:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765499.756564:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332608:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765499.756610:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765499.756669:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765499.756717:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765499/real 1775765499] req@ffffa0e9d0542a00 x1862023861332736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765499.756747:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861332736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765499.756765:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765499.756772:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765499.756778:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332736:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765500.331628:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765500.587614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765501.355719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765501.611671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765502.379611:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765502.635578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765503.403780:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765503.659626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765504.427701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765504.683604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765504.875627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765504.875642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765504.875650:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765504.875666:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1233 00000100:00080000:2.0:1775765504.875683:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765504.875693:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765504.875751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861332864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765504.875794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765504.875804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765504.875830:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765504.875851:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1233 00000100:00080000:2.0:1775765504.875858:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765504.875864:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765504.875873:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861332864:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775765504.875880:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765504.875901:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861332992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765504.875917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765504.875922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765504.875925:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765504.875930:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1233 00000100:00080000:2.0:1775765504.875935:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765504.875940:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765504.875947:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765504.875978:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765504.876008:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861333120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765504.876054:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765504/real 1775765504] req@ffffa0e9d0711500 x1862023861332864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765504.876072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861332864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765504.876082:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765504.876086:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765504.876089:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861332864:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765504.876118:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765504.876143:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765504.876157:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765504/real 1775765504] req@ffffa0e9d0713480 x1862023861332992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765504.876172:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861332992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765504.876180:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765504.876184:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765504.876188:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861332992:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765504.876195:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765504.876216:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765504.876226:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765504/real 1775765504] req@ffffa0e9d0712a00 x1862023861333120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765504.876265:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861333120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765504.876271:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765504.876273:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765504.876276:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333120:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765505.451670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765505.707624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765506.475634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765506.731640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765507.499832:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765507.755710:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765508.523572:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765508.779582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765509.547730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765509.803657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765509.995627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765509.995637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765509.995642:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765509.995652:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1238 00000100:00080000:0.0:1775765509.995664:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765509.995671:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765509.995747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861333248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765509.995839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765509.995844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765509.995847:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765509.995853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1238 00000100:00080000:0.0:1775765509.995859:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765509.995864:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765509.995876:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765509.995901:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861333376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765509.995980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765509.995983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765509.995987:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765509.995995:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1238 00000100:00100000:2.0:1775765509.996004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861333248:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765509.996005:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765509.996018:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765509.996026:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765509.996043:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861333504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765509.996117:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765509.996219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765509/real 1775765509] req@ffffa0e9d09df800 x1862023861333248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765509.996248:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861333248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765509.996259:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765509.996263:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765509.996267:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861333248:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765509.996275:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765509.996322:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765509.996332:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765509/real 1775765509] req@ffffa0e9d09dc700 x1862023861333376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765509.996341:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861333376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765509.996347:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765509.996349:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765509.996352:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333376:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765509.996377:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765509.996391:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765509.996398:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765509/real 1775765509] req@ffffa0e9d09dd880 x1862023861333504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765509.996407:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861333504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765509.996422:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765509.996425:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765509.996428:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333504:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765510.571803:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765510.827622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765511.595642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765511.851627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765512.619747:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765512.875575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765513.643741:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765513.899571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765514.667688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765514.923559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765515.115652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765515.115666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765515.115672:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765515.115686:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1243 00000100:00080000:3.0:1775765515.115701:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765515.115712:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765515.115774:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861333632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765515.115810:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765515.115815:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765515.115819:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765515.115825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1243 00000100:00080000:3.0:1775765515.115832:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765515.115838:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765515.115853:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765515.115897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861333760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765515.115898:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861333632:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765515.115917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765515.115922:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765515.115925:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765515.115931:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1243 00000100:00080000:3.0:1775765515.115936:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765515.115962:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765515.115970:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765515.115984:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:3.0:1775765515.115986:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861333888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765515.116052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765515/real 1775765515] req@ffffa0e9d0541500 x1862023861333632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765515.116092:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861333632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765515.116104:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765515.116108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765515.116113:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861333632:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765515.116143:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765515.116172:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765515.116186:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765515/real 1775765515] req@ffffa0e9d0541880 x1862023861333760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765515.116217:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861333760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765515.116240:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765515.116245:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765515.116248:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333760:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765515.116256:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765515.116280:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765515.116292:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765515/real 1775765515] req@ffffa0e9d0542680 x1862023861333888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765515.116304:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861333888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765515.116313:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765515.116316:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765515.116320:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861333888:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765515.691774:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765515.947595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765516.715710:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765516.971608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765517.739637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765517.995622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765518.763668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765519.019631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765519.787720:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765520.043614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765520.235607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765520.235617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765520.235623:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765520.235633:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1248 00000100:00080000:3.0:1775765520.235646:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765520.235652:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765520.235707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861334016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765520.235730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765520.235732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765520.235734:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765520.235736:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1248 00000100:00080000:3.0:1775765520.235739:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765520.235741:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765520.235750:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765520.235770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861334144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765520.235783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765520.235787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765520.235805:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765520.235823:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1248 00000100:00080000:3.0:1775765520.235826:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765520.235829:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765520.235835:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765520.235859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861334016:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775765520.235859:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861334272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765520.235970:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765520.236038:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765520/real 1775765520] req@ffffa0e9d0542680 x1862023861334016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765520.236072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861334016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765520.236082:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765520.236087:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765520.236091:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861334016:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765520.236099:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765520.236121:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765520.236134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765520/real 1775765520] req@ffffa0e9d0541880 x1862023861334144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765520.236149:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861334144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765520.236158:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765520.236163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765520.236167:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334144:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765520.236200:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765520.236222:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765520.236247:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765520/real 1775765520] req@ffffa0e9d0541500 x1862023861334272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765520.236279:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861334272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765520.236289:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765520.236294:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765520.236297:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334272:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765520.811636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765521.067601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765521.835707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765522.091610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765522.859921:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765523.115659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765523.883640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765524.139627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765524.907743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765525.163659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765525.355602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765525.355611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765525.355615:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765525.355624:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1253 00000100:00080000:0.0:1775765525.355636:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765525.355644:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765525.355684:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861334400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765525.355716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765525.355721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765525.355737:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765525.355742:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1253 00000100:00080000:0.0:1775765525.355788:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765525.355796:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765525.355810:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765525.355828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861334528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765525.355845:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765525.355849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765525.355852:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765525.355859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861334400:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765525.355860:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1253 00000100:00080000:0.0:1775765525.355866:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765525.355878:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765525.355884:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765525.355901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861334656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765525.355985:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765525.356051:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765525/real 1775765525] req@ffffa0e9d09dfb80 x1862023861334400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765525.356078:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861334400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765525.356093:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765525.356099:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765525.356105:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861334400:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765525.356138:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765525.356193:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765525.356215:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765525/real 1775765525] req@ffffa0e9d09df100 x1862023861334528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765525.356231:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861334528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765525.356241:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765525.356245:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765525.356249:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334528:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765525.356256:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765525.356275:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765525.356285:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765525/real 1775765525] req@ffffa0e9d09ded80 x1862023861334656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765525.356299:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861334656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765525.356308:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765525.356311:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765525.356315:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334656:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765525.931675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765526.187680:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765526.955628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765527.211598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765527.979633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765528.235600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765529.003654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765529.259571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765530.027578:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765530.283559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765530.475561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765530.475571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765530.475576:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765530.475587:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1258 00000100:00080000:2.0:1775765530.475607:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765530.475614:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765530.475671:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861334784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765530.475726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765530.475733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765530.475736:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765530.475743:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1258 00000100:00080000:2.0:1775765530.475750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765530.475756:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765530.475762:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861334784:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775765530.475772:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765530.475796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861334912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765530.475813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765530.475818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765530.475821:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775765530.475830:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:2.0:1775765530.475831:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1258 00000100:00080000:2.0:1775765530.475840:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765530.475861:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765530.475867:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765530.475890:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765530/real 1775765530] req@ffffa0e9d0710e00 x1862023861334784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765530.475901:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861335040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765530.475924:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861334784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765530.475941:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765530.475945:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765530.475949:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861334784:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765530.475987:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765530.476005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765530.476014:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765530/real 1775765530] req@ffffa0e9d0710a80 x1862023861334912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765530.476022:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861334912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765530.476028:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765530.476030:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765530.476032:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861334912:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765530.476036:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765530.476080:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765530.476090:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765530/real 1775765530] req@ffffa0e9d0710700 x1862023861335040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765530.476100:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861335040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765530.476107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765530.476110:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765530.476113:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335040:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765531.051733:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765531.307573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765532.075741:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765532.331573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765533.099821:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765533.355632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765534.123699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765534.379711:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765535.148021:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765535.403761:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765535.595655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765535.595691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765535.595709:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765535.595721:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1264 00000100:00080000:0.0:1775765535.595736:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765535.595744:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765535.595794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861335168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765535.595834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765535.595849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765535.595851:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765535.595858:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1264 00000100:00080000:0.0:1775765535.595863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765535.595869:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765535.595883:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765535.595902:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861335168:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765535.595903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861335296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765535.595931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765535.595935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765535.595938:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765535.595944:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1264 00000100:00080000:0.0:1775765535.595948:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765535.595978:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765535.595986:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765535.596002:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861335424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765535.596030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765535.596110:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765535/real 1775765535] req@ffffa0e9d09dc380 x1862023861335168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765535.596139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861335168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765535.596154:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765535.596160:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765535.596165:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861335168:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765535.596194:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765535.596223:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765535.596238:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765535/real 1775765535] req@ffffa0e9d09dc000 x1862023861335296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765535.596253:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861335296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765535.596264:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765535.596280:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765535.596284:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335296:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765535.596293:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765535.596319:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765535.596339:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765535/real 1775765535] req@ffffa0e9d09dce00 x1862023861335424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765535.596354:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861335424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765535.596364:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765535.596368:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765535.596371:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335424:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765536.171648:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765536.427592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765537.195795:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765537.451605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765538.219737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765538.475606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765539.243612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765539.499666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765540.267709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765540.523605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765540.715563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765540.715579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765540.715587:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765540.715603:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1269 00000100:00080000:0.0:1775765540.715638:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765540.715652:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765540.715709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861335552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765540.715752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765540.715757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765540.715760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765540.715765:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1269 00000100:00080000:0.0:1775765540.715771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765540.715776:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765540.715802:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765540.715829:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861335552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765540.715835:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861335680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765540.715864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765540.715872:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765540.715900:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765540.715908:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1269 00000100:00080000:0.0:1775765540.715916:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765540.715921:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775765540.715928:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765540.715938:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765540.715957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861335808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765540.716022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765540/real 1775765540] req@ffffa0e9d09de300 x1862023861335552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765540.716076:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861335552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765540.716124:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765540.716134:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765540.716143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861335552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765540.716182:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765540.716220:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765540.716238:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765540/real 1775765540] req@ffffa0e9d09dea00 x1862023861335680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765540.716270:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861335680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765540.716283:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765540.716287:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765540.716292:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335680:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765540.716300:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765540.716323:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765540.716336:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765540/real 1775765540] req@ffffa0e9d09df480 x1862023861335808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765540.716351:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861335808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765540.716361:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765540.716364:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765540.716368:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861335808:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765541.291707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765541.547649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765542.315675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765542.571620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765543.339660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765543.595567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765544.363646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765544.619616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765545.387737:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765545.643610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765545.835742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765545.835755:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765545.835762:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765545.835773:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1274 00000100:00080000:3.0:1775765545.835788:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765545.835797:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765545.835855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861335936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765545.835921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765545.835952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765545.835957:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765545.835966:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1274 00000100:00080000:3.0:1775765545.835975:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765545.835983:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765545.836000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861335936:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775765545.836001:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765545.836033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861336064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765545.836055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765545.836061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765545.836064:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765545.836072:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1274 00000100:00080000:3.0:1775765545.836079:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765545.836085:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765545.836094:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765545.836116:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861336192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765545.836129:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765545.836219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765545/real 1775765545] req@ffffa0e9d0542a00 x1862023861335936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765545.836255:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861335936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765545.836269:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765545.836275:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765545.836280:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861335936:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765545.836329:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765545.836380:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765545.836396:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765545/real 1775765545] req@ffffa0e9d0540000 x1862023861336064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765545.836411:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861336064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765545.836421:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765545.836424:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765545.836440:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336064:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765545.836448:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765545.836490:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765545.836503:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765545/real 1775765545] req@ffffa0e9d0543b80 x1862023861336192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765545.836518:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861336192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765545.836527:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765545.836610:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765545.836617:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336192:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765546.411727:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765546.667680:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765547.435684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765547.691649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765548.459696:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765548.715688:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765549.483688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765549.739626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765550.507830:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765550.763676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765550.955654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765550.955668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765550.955676:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765550.955691:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1279 00000100:00080000:3.0:1775765550.955708:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765550.955717:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765550.955819:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861336320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765550.955866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765550.955871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765550.955874:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765550.955879:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1279 00000100:00080000:3.0:1775765550.955885:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765550.955890:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765550.955906:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765550.955923:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861336448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765550.955937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765550.955943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765550.955945:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765550.955950:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1279 00000100:00080000:3.0:1775765550.955953:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765550.955957:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765550.955963:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765550.955978:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861336320:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775765550.955989:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861336576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765550.956103:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765550.956176:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765550/real 1775765550] req@ffffa0e9d0541c00 x1862023861336320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765550.956213:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861336320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765550.956270:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765550.956281:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765550.956290:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861336320:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765550.956304:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765550.956353:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765550.956372:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765550/real 1775765550] req@ffffa0e9d0540380 x1862023861336448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765550.956388:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861336448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765550.956398:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765550.956403:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765550.956407:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336448:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765550.956451:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765550.956473:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765550.956577:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765550/real 1775765550] req@ffffa0e9d0542300 x1862023861336576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765550.956598:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861336576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765550.956609:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765550.956613:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765550.956617:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336576:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765551.531690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765551.787596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765552.555741:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765552.811623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765553.579668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765553.835587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765554.603652:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765554.859655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765555.627740:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765555.883645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765556.075712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765556.075726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765556.075733:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765556.075748:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1284 00000100:00080000:0.0:1775765556.075765:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765556.075775:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765556.075841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861336704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765556.075886:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765556.075891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765556.075893:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765556.075897:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1284 00000100:00080000:0.0:1775765556.075901:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765556.075932:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765556.075948:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765556.075964:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861336832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765556.075977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765556.075982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765556.075985:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765556.075990:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1284 00000100:00080000:0.0:1775765556.075995:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765556.075999:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765556.076004:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765556.076016:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861336960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765556.076034:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861336704:192.168.202.104@tcp:250:kworker.0 00000800:00000100:2.0:1775765556.076134:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765556.076209:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765556/real 1775765556] req@ffffa0e9d09df480 x1862023861336704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765556.076242:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861336704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765556.076257:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765556.076262:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765556.076268:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861336704:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765556.076281:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765556.076324:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765556.076348:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765556/real 1775765556] req@ffffa0e9d09dea00 x1862023861336832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765556.076384:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861336832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765556.076401:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765556.076407:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765556.076432:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336832:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765556.076546:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765556.076581:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765556.076598:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765556/real 1775765556] req@ffffa0e9d09de300 x1862023861336960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765556.076613:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861336960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765556.076623:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765556.076628:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765556.076632:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861336960:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765556.651742:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765556.907629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765557.675776:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765557.931669:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765558.699654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765558.955622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765559.723641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765559.979566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765560.747681:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765561.003693:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765561.195625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765561.195636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765561.195642:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765561.195652:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1289 00000100:00080000:0.0:1775765561.195667:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765561.195686:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765561.195733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861337088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765561.195760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765561.195783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765561.195786:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765561.195797:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1289 00000100:00100000:2.0:1775765561.195799:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861337088:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765561.195804:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765561.195818:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765561.195832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765561.195848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861337216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765561.195864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765561.195868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765561.195870:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765561.195874:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1289 00000100:00080000:0.0:1775765561.195878:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765561.195882:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765561.195889:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765561.195899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861337344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765561.196001:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337216:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765561.196038:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337344:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765561.771665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765562.027606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765562.795617:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765563.051581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765563.819646:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765564.075596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765564.267830:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765564.267883:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765564.267907:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765564.267982:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765564.268000:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765564.268235:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765561/real 1775765564] req@ffffa0e9d09dc000 x1862023861337216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765564.268281:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861337216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765564.268302:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765564.268310:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765564.268319:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337216:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765564.268333:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765561/real 1775765564] req@ffffa0e9d09dc380 x1862023861337344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765564.268356:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861337344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765564.268369:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765564.268375:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765564.268381:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337344:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765564.268418:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765561/real 1775765564] req@ffffa0e9d09dce00 x1862023861337088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765564.268442:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861337088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765564.268457:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765564.268463:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765564.268469:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861337088:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775765564.843692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765565.099598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765565.867637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765566.123595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765566.315610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765566.315619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765566.315624:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765566.315634:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1294 00000100:00080000:2.0:1775765566.315644:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765566.315650:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765566.315682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861337472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765566.315707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765566.315710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765566.315711:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765566.315714:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1294 00000100:00080000:2.0:1775765566.315716:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765566.315718:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765566.315727:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765566.315736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861337600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765566.315743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765566.315745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765566.315747:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765566.315751:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1294 00000100:00080000:2.0:1775765566.315752:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765566.315754:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765566.315767:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765566.315769:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861337472:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765566.315786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861337728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765566.315875:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765566.315940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765566/real 1775765566] req@ffffa0e9d0712680 x1862023861337472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765566.315988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861337472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765566.316005:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765566.316011:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765566.316016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861337472:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765566.316027:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765566.316067:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765566.316082:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765566/real 1775765566] req@ffffa0e9d0712d80 x1862023861337600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765566.316093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861337600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765566.316101:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765566.316106:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765566.316109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337600:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765566.316145:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765566.316166:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765566.316177:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765566/real 1775765566] req@ffffa0e9d0711f80 x1862023861337728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765566.316187:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861337728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765566.316214:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765566.316221:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765566.316226:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337728:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765566.891617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765567.147626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765567.915678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765568.171644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765568.939612:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765569.195600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765569.963621:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765570.219632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765570.987675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765571.243585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765571.435599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765571.435609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765571.435614:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765571.435625:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1299 00000100:00080000:0.0:1775765571.435638:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765571.435646:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765571.435702:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861337856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765571.435732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765571.435736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765571.435738:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765571.435743:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1299 00000100:00080000:0.0:1775765571.435747:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765571.435752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765571.435766:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765571.435775:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861337856:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765571.435779:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861337984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765571.435790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765571.435815:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765571.435817:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765571.435821:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1299 00000100:00080000:0.0:1775765571.435825:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765571.435829:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765571.435834:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765571.435844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861338112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765571.435867:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765571.435925:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765571/real 1775765571] req@ffffa0e9d09ded80 x1862023861337856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765571.435948:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861337856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765571.435961:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765571.435965:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765571.435967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861337856:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765571.435992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765571.436013:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765571.436022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765571/real 1775765571] req@ffffa0e9d09df100 x1862023861337984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765571.436030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861337984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765571.436042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765571.436044:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765571.436047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861337984:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765571.436051:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765571.436065:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765571.436072:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765571/real 1775765571] req@ffffa0e9d09dfb80 x1862023861338112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765571.436078:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861338112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765571.436083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765571.436085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765571.436087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338112:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765572.011675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765572.267620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765573.035671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765573.291719:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765574.059650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765574.315691:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765575.083623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765575.339596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765576.107604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765576.363561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765576.555587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765576.555595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765576.555599:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765576.555607:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1305 00000100:00080000:2.0:1775765576.555616:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765576.555637:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765576.555673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861338240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765576.555705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765576.555708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765576.555710:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765576.555713:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1305 00000100:00080000:2.0:1775765576.555717:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765576.555719:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765576.555728:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765576.555739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861338368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765576.555748:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861338240:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765576.555750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765576.555756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765576.555757:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765576.555761:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1305 00000100:00080000:2.0:1775765576.555763:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765576.555765:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765576.555768:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765576.555776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861338496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765576.555811:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765576.555843:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765576/real 1775765576] req@ffffa0e9d0711c00 x1862023861338240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765576.555856:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861338240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765576.555864:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765576.555867:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765576.555870:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861338240:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765576.555891:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765576.555913:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765576.555927:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765576/real 1775765576] req@ffffa0e9d0713b80 x1862023861338368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765576.555960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861338368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765576.555966:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765576.555968:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765576.555970:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338368:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765576.555974:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765576.555991:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765576.555998:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765576/real 1775765576] req@ffffa0e9d0710000 x1862023861338496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765576.556006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861338496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765576.556011:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765576.556013:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765576.556015:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338496:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765577.131649:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765577.387577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765578.155694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765578.411626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765579.179675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765579.435570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765580.203708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765580.459542:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765581.227689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765581.483595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765581.675668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765581.675681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765581.675687:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765581.675699:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1310 00000100:00080000:0.0:1775765581.675712:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765581.675720:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765581.675782:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861338624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765581.675830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765581.675833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765581.675835:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765581.675839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1310 00000100:00080000:0.0:1775765581.675843:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765581.675848:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765581.675857:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765581.675871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861338752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765581.675875:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861338624:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765581.675886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765581.675889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765581.675890:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765581.675894:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1310 00000100:00080000:0.0:1775765581.675912:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765581.675915:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765581.675920:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765581.675931:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861338880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765581.675975:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765581.676017:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765581/real 1775765581] req@ffffa0e9d09dd880 x1862023861338624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765581.676035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861338624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765581.676046:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765581.676050:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765581.676054:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861338624:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765581.676075:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765581.676095:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765581.676106:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765581/real 1775765581] req@ffffa0e9d09dc700 x1862023861338752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765581.676134:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861338752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765581.676141:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765581.676144:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765581.676147:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338752:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765581.676161:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765581.676178:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765581.676186:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765581/real 1775765581] req@ffffa0e9d09df800 x1862023861338880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765581.676196:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861338880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765581.676202:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765581.676204:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765581.676206:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861338880:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765582.251703:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765582.507572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765583.275688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765583.531594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765584.299629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765584.555615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765585.323648:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765585.579609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765586.347820:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765586.603568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765586.795587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765586.795601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765586.795627:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765586.795640:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1315 00000100:00080000:0.0:1775765586.795653:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765586.795662:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765586.795703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861339008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765586.795753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765586.795758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765586.795761:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765586.795767:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1315 00000100:00080000:0.0:1775765586.795772:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765586.795777:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765586.795788:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765586.795806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861339136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765586.795823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765586.795828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765586.795831:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765586.795838:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1315 00000100:00080000:0.0:1775765586.795856:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765586.795866:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765586.795874:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861339008:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775765586.795879:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765586.795904:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861339264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765586.796016:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765586.796083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765586/real 1775765586] req@ffffa0e9d09dd180 x1862023861339008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765586.796110:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861339008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765586.796141:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765586.796226:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765586.796231:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861339008:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765586.796240:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765586.796270:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765586.796283:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765586/real 1775765586] req@ffffa0e9d09ddc00 x1862023861339136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765586.796296:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861339136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765586.796304:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765586.796307:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765586.796311:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339136:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765586.796342:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765586.796358:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765586.796367:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765586/real 1775765586] req@ffffa0e9d09dd500 x1862023861339264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765586.796383:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861339264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765586.796395:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765586.796400:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765586.796405:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339264:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765587.371614:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765587.627666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765588.395680:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765588.651556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765589.419748:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765589.675617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765590.443683:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765590.699564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765591.467708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765591.723639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765591.915573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765591.915585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765591.915591:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765591.915602:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1320 00000100:00080000:0.0:1775765591.915616:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765591.915625:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765591.915668:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861339392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765591.915700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765591.915704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765591.915707:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765591.915712:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1320 00000100:00080000:0.0:1775765591.915718:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765591.915734:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765591.915747:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765591.915763:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861339392:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765591.915769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861339520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765591.915783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765591.915786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765591.915788:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765591.915792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1320 00000100:00080000:0.0:1775765591.915811:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765591.915815:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765591.915821:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765591.915834:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765591.915836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861339648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765591.915894:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765591/real 1775765591] req@ffffa0e9d09ddf80 x1862023861339392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765591.915911:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861339392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765591.915921:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765591.915924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765591.915928:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861339392:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765591.915951:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765591.915968:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765591.915978:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765591/real 1775765591] req@ffffa0e9d09dca80 x1862023861339520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765591.915988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861339520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765591.915994:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765591.915997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765591.916000:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339520:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765591.916014:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765591.916031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765591.916039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765591/real 1775765591] req@ffffa0e9d09de680 x1862023861339648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765591.916054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861339648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765591.916064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765591.916068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765591.916072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339648:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765592.491595:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765592.747554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765593.515651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765593.771592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765594.539603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765594.795590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765595.563694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765595.819656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765596.587685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765596.843597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765597.035767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765597.035779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765597.035785:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765597.035796:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1325 00000100:00080000:2.0:1775765597.035809:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765597.035816:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765597.035861:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861339776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765597.035930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765597.035934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765597.035936:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765597.035941:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1325 00000100:00080000:2.0:1775765597.035947:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765597.035952:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765597.035964:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765597.035982:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861339904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765597.035998:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861339776:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765597.036003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765597.036020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765597.036023:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765597.036031:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1325 00000100:00080000:2.0:1775765597.036038:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765597.036047:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765597.036057:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765597.036109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861340032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765597.036153:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765597.036214:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765597/real 1775765597] req@ffffa0e9d0710380 x1862023861339776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765597.036236:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861339776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765597.036248:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765597.036253:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765597.036257:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861339776:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765597.036300:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765597.036321:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765597.036333:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765597/real 1775765597] req@ffffa0e9d0711180 x1862023861339904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765597.036343:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861339904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765597.036373:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765597.036376:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765597.036379:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861339904:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765597.036385:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765597.036416:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765597.036427:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765597/real 1775765597] req@ffffa0e9d0710000 x1862023861340032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765597.036439:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861340032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765597.036451:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765597.036456:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765597.036460:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340032:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765597.611597:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765597.867639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765598.635677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765598.891582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765599.659627:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765599.915564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765600.683653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765600.939574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765601.707663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765601.963574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765602.155610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765602.155621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765602.155627:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765602.155638:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1330 00000100:00080000:2.0:1775765602.155652:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765602.155660:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765602.155704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861340160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765602.155739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765602.155743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765602.155745:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765602.155749:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1330 00000100:00080000:2.0:1775765602.155753:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765602.155757:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765602.155768:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765602.155782:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861340288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765602.155795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765602.155803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861340160:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765602.155803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765602.155806:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765602.155819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1330 00000100:00080000:2.0:1775765602.155823:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765602.155826:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765602.155853:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765602.155868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861340416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765602.155893:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765602.155946:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765602/real 1775765602] req@ffffa0e9d0713b80 x1862023861340160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765602.155968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861340160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765602.155981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765602.155986:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765602.156001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861340160:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765602.156048:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765602.156077:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765602.156092:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765602/real 1775765602] req@ffffa0e9d0711c00 x1862023861340288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765602.156106:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861340288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765602.156115:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765602.156119:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765602.156122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340288:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765602.156129:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765602.156145:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765602.156154:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765602/real 1775765602] req@ffffa0e9d0711f80 x1862023861340416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765602.156166:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861340416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765602.156174:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765602.156177:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765602.156180:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340416:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765602.731711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765602.987592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765603.755598:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765604.011580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765604.779672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765605.035565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765605.805750:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765606.059575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765606.827586:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765607.083551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765607.275563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765607.275577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765607.275581:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765607.275590:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1335 00000100:00080000:0.0:1775765607.275599:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765607.275605:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765607.275635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861340544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765607.275659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765607.275661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765607.275663:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765607.275665:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1335 00000100:00080000:0.0:1775765607.275668:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765607.275671:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765607.275679:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765607.275688:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861340672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765607.275692:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861340544:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765607.275699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765607.275702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765607.275703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765607.275706:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1335 00000100:00080000:0.0:1775765607.275708:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765607.275709:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765607.275713:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765607.275721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861340800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765607.275753:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765607.275786:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765607/real 1775765607] req@ffffa0e9d09de680 x1862023861340544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765607.275801:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861340544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765607.275809:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765607.275813:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765607.275816:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861340544:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765607.275834:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765607.275848:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765607.275856:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765607/real 1775765607] req@ffffa0e9d09dca80 x1862023861340672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765607.275865:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861340672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765607.275870:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765607.275879:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765607.275881:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340672:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765607.275885:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765607.275898:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765607.275905:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765607/real 1775765607] req@ffffa0e9d09ddf80 x1862023861340800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765607.275912:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861340800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765607.275917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765607.275919:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765607.275921:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861340800:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765607.851644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765608.107676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765608.875764:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765609.131676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765609.899618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765610.155603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765610.923703:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765611.179636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765611.947654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765612.203566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765612.395635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765612.395649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765612.395656:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765612.395690:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1340 00000100:00080000:3.0:1775765612.397368:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765612.397413:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765612.397511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861340928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765612.397546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765612.397551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765612.397553:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765612.397559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1340 00000100:00080000:3.0:1775765612.397566:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765612.397572:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765612.397585:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765612.397605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861341056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765612.397621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765612.397628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765612.397630:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861340928:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765612.397632:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765612.397646:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1340 00000100:00080000:3.0:1775765612.397653:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765612.397659:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765612.397669:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765612.397695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861341184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765612.397730:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765612.397796:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765612/real 1775765612] req@ffffa0e9d0543800 x1862023861340928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765612.397821:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861340928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765612.397857:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765612.397863:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765612.397869:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861340928:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765612.397897:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765612.397954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765612.397973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765612/real 1775765612] req@ffffa0e9d0540700 x1862023861341056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765612.397989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861341056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765612.397999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765612.398004:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765612.398008:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341056:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765612.398015:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765612.398038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765612.398049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765612/real 1775765612] req@ffffa0e9d0540e00 x1862023861341184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765612.398063:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861341184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765612.398073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765612.398076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765612.398079:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341184:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765612.971644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765613.227629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765613.995644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765614.251637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765615.019660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765615.275654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765616.043621:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765616.299554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765617.067767:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765617.323594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765617.515677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765617.515686:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765617.515691:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765617.515719:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1346 00000100:00080000:1.0:1775765617.515731:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765617.515738:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765617.515794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861341312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765617.515843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765617.515847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765617.515849:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765617.515852:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1346 00000100:00080000:1.0:1775765617.515855:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765617.515858:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765617.515873:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765617.515906:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861341440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765617.515915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765617.515935:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765617.515939:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765617.515942:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861341312:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765617.515945:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1346 00000100:00080000:1.0:1775765617.515953:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765617.515960:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765617.515966:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765617.515978:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861341568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765617.516049:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765617.516118:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765617/real 1775765617] req@ffffa0e9c8ded500 x1862023861341312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765617.516143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861341312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765617.516156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765617.516162:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765617.516167:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861341312:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765617.516177:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765617.516214:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765617.516229:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765617/real 1775765617] req@ffffa0e9c8deed80 x1862023861341440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765617.516242:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861341440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765617.516251:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765617.516255:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765617.516272:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341440:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765617.516304:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765617.516331:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765617.516344:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765617/real 1775765617] req@ffffa0e9c8dec380 x1862023861341568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765617.516357:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861341568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765617.516365:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765617.516368:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765617.516372:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341568:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765618.091659:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765618.347587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765619.115637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765619.371674:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765620.139681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765620.395624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765621.163678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765621.419676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765622.187712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765622.443605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765622.635656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765622.635667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765622.635672:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765622.635682:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1351 00000100:00080000:1.0:1775765622.635693:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765622.635700:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765622.635769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861341696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765622.635802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765622.635806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765622.635808:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765622.635812:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1351 00000100:00080000:1.0:1775765622.635815:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765622.635819:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765622.635883:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765622.635897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861341824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765622.635910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765622.635915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765622.635917:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861341696:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765622.635918:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765622.635927:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1351 00000100:00080000:1.0:1775765622.635935:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765622.635938:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765622.635943:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765622.635955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861341952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765622.636007:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765622.636067:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765622/real 1775765622] req@ffffa0e9c8dee300 x1862023861341696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765622.636089:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861341696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765622.636099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765622.636111:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765622.636116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861341696:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765622.636140:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765622.636195:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765622.636208:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765622/real 1775765622] req@ffffa0e9c8dec000 x1862023861341824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765622.636220:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861341824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765622.636228:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765622.636231:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765622.636234:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341824:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765622.636239:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765622.636255:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765622.636287:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765622/real 1775765622] req@ffffa0e9c8ded180 x1862023861341952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765622.636297:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861341952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765622.636304:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765622.636306:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765622.636309:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861341952:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765623.211718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765623.467561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765624.235762:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765624.491659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765625.259676:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765625.515631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765626.283743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765626.539654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765627.307851:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765627.563610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765627.755621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765627.755633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765627.755639:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765627.755648:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1356 00000100:00080000:2.0:1775765627.755658:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765627.755663:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765627.755708:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861342080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765627.755739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765627.755743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765627.755746:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765627.755750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1356 00000100:00080000:2.0:1775765627.755755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765627.755760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765627.755770:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765627.755798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861342208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765627.755809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765627.755812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765627.755839:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765627.755845:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861342080:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765627.755846:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1356 00000100:00080000:2.0:1775765627.755854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765627.755858:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765627.755864:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765627.755883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861342336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765627.755945:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342208:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765627.755977:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342336:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765628.331660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765628.587682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765629.355754:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765629.611649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765630.379841:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765630.635681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775765630.827861:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775765630.827917:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765630.827936:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765630.827979:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765630.827987:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765630.828044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765627/real 1775765630] req@ffffa0e9d0710700 x1862023861342336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765630.828080:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861342336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765630.828095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765630.828101:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765630.828107:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765630.828117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765627/real 1775765630] req@ffffa0e9d0712d80 x1862023861342080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765630.828134:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861342080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765630.828143:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765630.828147:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765630.828152:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861342080:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765630.828160:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765627/real 1775765630] req@ffffa0e9d0712680 x1862023861342208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765630.828222:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861342208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765630.828232:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765630.828237:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765630.828241:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342208:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765631.403642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765631.659646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765632.427633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765632.683613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765632.875789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765632.875803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765632.875809:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765632.875823:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1361 00000100:00080000:2.0:1775765632.875840:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765632.875849:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765632.875903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861342464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765632.875970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765632.875977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765632.875981:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765632.875988:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1361 00000100:00080000:2.0:1775765632.875996:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765632.876004:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765632.876022:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765632.876049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861342592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765632.876068:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861342464:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765632.876074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765632.876087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765632.876091:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765632.876099:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1361 00000100:00080000:2.0:1775765632.876107:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765632.876115:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765632.876124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765632.876149:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861342720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765632.876228:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765632.876365:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765632/real 1775765632] req@ffffa0e9d0710a80 x1862023861342464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765632.876405:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861342464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765632.876426:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765632.876434:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765632.876443:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861342464:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765632.876488:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765632.876587:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765632.876611:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765632/real 1775765632] req@ffffa0e9d0710e00 x1862023861342592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765632.876660:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861342592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765632.876701:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765632.876708:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765632.876715:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342592:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765632.876726:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765632.876797:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765632.876817:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765632/real 1775765632] req@ffffa0e9d0712a00 x1862023861342720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765632.876840:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861342720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765632.876855:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765632.876861:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765632.876866:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342720:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765633.451768:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765633.707651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765634.475672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765634.731774:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765635.499855:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765635.755629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765636.523807:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765636.779608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765637.547641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765637.803615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765637.995680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765637.995723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765637.995730:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765637.995745:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1366 00000100:00080000:2.0:1775765637.995761:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765637.995771:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765637.995824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861342848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765637.995876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765637.995882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765637.995885:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765637.995892:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1366 00000100:00080000:2.0:1775765637.995898:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765637.995906:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765637.995922:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765637.995945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861342976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765637.995969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765637.995975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765637.995979:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765637.995990:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1366 00000100:00100000:0.0:1775765637.995992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861342848:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765637.995997:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765637.996031:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765637.996066:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765637.996095:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861343104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765637.996111:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765637.996207:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765637/real 1775765637] req@ffffa0e9d0713480 x1862023861342848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765637.996231:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861342848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765637.996244:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765637.996249:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765637.996271:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861342848:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765637.996302:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765637.996326:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765637.996338:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765637/real 1775765637] req@ffffa0e9d0711500 x1862023861342976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765637.996348:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861342976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765637.996356:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765637.996359:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765637.996362:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861342976:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765637.996367:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765637.996380:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765637.996404:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765637/real 1775765637] req@ffffa0e9d0712300 x1862023861343104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765637.996415:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861343104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765637.996421:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765637.996423:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765637.996426:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343104:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765638.571835:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765638.827660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765639.595798:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765639.851637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765640.619725:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765640.875625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765641.643675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765641.899559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765642.667675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765642.923602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765643.115593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765643.115601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765643.115621:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765643.115631:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1371 00000100:00080000:0.0:1775765643.115668:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765643.115676:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765643.115712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861343232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765643.115739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765643.115742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765643.115743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765643.115746:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1371 00000100:00080000:0.0:1775765643.115749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765643.115751:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765643.115759:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765643.115768:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861343360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765643.115775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765643.115778:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765643.115779:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765643.115782:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1371 00000100:00080000:0.0:1775765643.115784:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765643.115789:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765643.115790:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861343232:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775765643.115792:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765643.115804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861343488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765643.115862:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765643.115916:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765643/real 1775765643] req@ffffa0e9d09dd500 x1862023861343232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765643.115931:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861343232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765643.115939:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765643.115942:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765643.115945:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861343232:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765643.115950:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765643.115969:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765643.115992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765643/real 1775765643] req@ffffa0e9d09ddc00 x1862023861343360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765643.116005:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861343360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765643.116023:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765643.116027:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765643.116030:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343360:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765643.116052:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765643.116069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765643.116085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765643/real 1775765643] req@ffffa0e9d09dd180 x1862023861343488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765643.116093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861343488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765643.116099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765643.116101:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765643.116103:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343488:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765643.691663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765643.947671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765644.715710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765644.971583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765645.739665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765645.995719:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765646.763712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765647.019593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765647.787753:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765648.043631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765648.235725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765648.235736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765648.235742:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765648.235753:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1376 00000100:00080000:2.0:1775765648.235798:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765648.235806:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765648.235843:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861343616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765648.235881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765648.235884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765648.235887:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765648.235891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1376 00000100:00080000:2.0:1775765648.235895:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765648.235898:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765648.235919:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765648.235938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861343744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765648.235939:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861343616:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765648.235962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765648.235966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765648.235968:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765648.235973:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1376 00000100:00080000:2.0:1775765648.235977:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765648.235980:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765648.235986:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765648.236011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861343872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765648.236039:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765648.236088:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765648/real 1775765648] req@ffffa0e9d0711880 x1862023861343616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765648.236108:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861343616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765648.236120:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765648.236124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765648.236128:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861343616:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765648.236155:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765648.236178:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765648.236191:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765648/real 1775765648] req@ffffa0e9d0713100 x1862023861343744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765648.236204:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861343744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765648.236212:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765648.236216:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765648.236219:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343744:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765648.236225:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765648.236241:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765648.236264:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765648/real 1775765648] req@ffffa0e9d0712300 x1862023861343872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765648.236277:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861343872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765648.236285:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765648.236288:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765648.236292:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861343872:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765648.811678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765649.067553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765649.835768:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765650.091641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765650.859734:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765651.115595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765651.883682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765652.139634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765652.907669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765653.163662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765653.355606:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765653.355616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765653.355622:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765653.355633:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1381 00000100:00080000:2.0:1775765653.355644:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765653.355650:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765653.355687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861344000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765653.355719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765653.355723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765653.355725:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765653.355730:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1381 00000100:00080000:2.0:1775765653.355735:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765653.355741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765653.355753:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765653.355768:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861344128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765653.355769:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861344000:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765653.355798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765653.355801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765653.355803:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765653.355823:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1381 00000100:00080000:2.0:1775765653.355830:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765653.355836:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765653.355843:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765653.355857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861344256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765653.355867:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765653.355911:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765653/real 1775765653] req@ffffa0e9d0711500 x1862023861344000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765653.355928:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861344000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765653.355937:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765653.355941:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765653.355945:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861344000:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765653.355972:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765653.355988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765653.355997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765653/real 1775765653] req@ffffa0e9d0713480 x1862023861344128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765653.356006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861344128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765653.356028:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765653.356030:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765653.356033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765653.356037:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765653.356052:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765653.356059:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765653/real 1775765653] req@ffffa0e9d0712a00 x1862023861344256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765653.356067:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861344256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765653.356072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765653.356075:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765653.356077:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344256:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765653.931802:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765654.187650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765654.955698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765655.211701:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765655.979685:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765656.235671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765657.003759:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765657.259653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765658.027801:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765658.283644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765658.475665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765658.475686:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765658.475695:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765658.475717:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1386 00000100:00080000:2.0:1775765658.475739:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765658.475753:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765658.475829:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861344384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765658.475878:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765658.475885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765658.475889:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765658.475898:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1386 00000100:00080000:2.0:1775765658.475906:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765658.475929:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861344384:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765658.475945:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765658.475967:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765658.475996:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861344512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765658.476017:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765658.476024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765658.476028:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765658.476035:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1386 00000100:00080000:2.0:1775765658.476044:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765658.476045:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:2.0:1775765658.476060:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765658.476072:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765658.476095:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861344640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765658.476114:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765658/real 1775765658] req@ffffa0e9d0710e00 x1862023861344384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765658.476142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861344384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765658.476156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765658.476161:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765658.476180:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861344384:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765658.476217:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765658.476246:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765658.476262:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765658/real 1775765658] req@ffffa0e9d0710a80 x1862023861344512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765658.476277:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861344512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765658.476286:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765658.476290:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765658.476294:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344512:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765658.476300:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765658.476318:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765658.476327:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765658/real 1775765658] req@ffffa0e9d0712680 x1862023861344640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765658.476340:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861344640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765658.476349:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765658.476352:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765658.476355:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344640:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765659.051604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765659.307642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765660.075716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765660.331658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765661.099736:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765661.355607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765662.123609:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765662.379596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765663.147693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765663.403621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765663.595764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765663.595778:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765663.595807:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765663.595819:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1392 00000100:00080000:0.0:1775765663.595833:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765663.595841:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765663.595890:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861344768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765663.595951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765663.595955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765663.595958:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765663.595963:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1392 00000100:00080000:0.0:1775765663.595969:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765663.595974:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765663.595987:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765663.596001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861344896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765663.596017:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765663.596027:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765663.596029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861344768:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765663.596031:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765663.596051:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1392 00000100:00080000:0.0:1775765663.596056:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765663.596060:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765663.596066:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765663.596081:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861345024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765663.596161:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765663.596235:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765663/real 1775765663] req@ffffa0e9d09df800 x1862023861344768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765663.596267:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861344768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765663.596307:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765663.596326:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765663.596334:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861344768:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765663.596379:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765663.596421:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765663.596459:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765663/real 1775765663] req@ffffa0e9d09dc700 x1862023861344896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765663.596473:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861344896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765663.596482:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765663.596523:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765663.596529:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861344896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765663.596540:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765663.596576:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765663.596590:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765663/real 1775765663] req@ffffa0e9d09dd880 x1862023861345024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765663.596602:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861345024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765663.596609:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765663.596613:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765663.596616:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345024:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765664.171677:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765664.427562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765665.195708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765665.451651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765666.219716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765666.475599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765667.243760:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765667.499625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765668.267611:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765668.523614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765668.715608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765668.715622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765668.715628:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765668.715640:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1397 00000100:00080000:0.0:1775765668.715654:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765668.715662:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765668.715711:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861345152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765668.715752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765668.715757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765668.715760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765668.715765:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1397 00000100:00080000:0.0:1775765668.715769:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765668.715774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765668.715786:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765668.715800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861345280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765668.715814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765668.715819:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765668.715821:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765668.715825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1397 00000100:00080000:0.0:1775765668.715830:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765668.715834:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765668.715877:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765668.715881:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861345152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765668.715897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861345408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765668.716015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765668.716109:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765668/real 1775765668] req@ffffa0e9d09dfb80 x1862023861345152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765668.716133:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861345152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765668.716145:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765668.716150:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765668.716154:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861345152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765668.716163:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765668.716202:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765668.716221:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765668/real 1775765668] req@ffffa0e9d09df100 x1862023861345280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765668.716240:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861345280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765668.716252:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765668.716256:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765668.716278:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345280:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765668.716318:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765668.716364:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765668.716378:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765668/real 1775765668] req@ffffa0e9d09ded80 x1862023861345408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765668.716410:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861345408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765668.716423:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765668.716428:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765668.716433:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345408:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765669.291750:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765669.547635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765670.315788:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765670.571595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765671.339634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765671.595650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765672.363701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765672.619597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765673.387740:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765673.643721:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765673.835666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765673.835683:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765673.835702:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765673.835718:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1402 00000100:00080000:0.0:1775765673.835738:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765673.835750:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765673.835846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861345536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765673.835879:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765673.835884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765673.835886:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765673.835892:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1402 00000100:00080000:0.0:1775765673.835897:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765673.835902:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765673.835916:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765673.835926:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861345536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765673.835936:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861345664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765673.835957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765673.835961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765673.835962:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765673.835966:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1402 00000100:00080000:0.0:1775765673.835996:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765673.836002:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765673.836008:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765673.836022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861345792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765673.836059:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765673.836131:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765673/real 1775765673] req@ffffa0e9d09dce00 x1862023861345536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765673.836157:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861345536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765673.836186:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765673.836191:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765673.836197:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861345536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765673.836231:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765673.836276:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765673.836298:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765673/real 1775765673] req@ffffa0e9d09dc380 x1862023861345664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765673.836320:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861345664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765673.836335:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765673.836379:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765673.836387:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765673.836397:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765673.836430:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765673.836446:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765673/real 1775765673] req@ffffa0e9d09dc000 x1862023861345792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765673.836464:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861345792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765673.836480:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765673.836487:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765673.836493:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861345792:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765674.411638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765674.667637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765675.435669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765675.691634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765676.459806:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765676.715682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765677.483638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765677.739590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765678.507693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765678.763593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765678.955683:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765678.955697:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765678.955704:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765678.955718:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1407 00000100:00080000:2.0:1775765678.955735:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765678.955744:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765678.955802:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861345920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765678.955846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765678.955851:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765678.955853:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765678.955858:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1407 00000100:00080000:2.0:1775765678.955862:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765678.955866:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765678.955880:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765678.955895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861346048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765678.955909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765678.955921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861345920:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765678.955938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765678.955941:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765678.955948:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1407 00000100:00080000:2.0:1775765678.955954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765678.955959:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765678.955966:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765678.955983:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861346176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765678.956024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765678.956085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765678/real 1775765678] req@ffffa0e9d0712d80 x1862023861345920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765678.956110:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861345920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765678.956124:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765678.956129:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765678.956134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861345920:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765678.956166:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765678.956192:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765678.956206:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765678/real 1775765678] req@ffffa0e9d0710700 x1862023861346048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765678.956219:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861346048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765678.956229:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765678.956233:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765678.956249:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765678.956257:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765678.956281:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765678.956320:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765678/real 1775765678] req@ffffa0e9d0711f80 x1862023861346176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765678.956335:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861346176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765678.956345:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765678.956348:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765678.956352:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346176:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765679.531701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765679.787637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765680.555664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765680.811668:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765681.579744:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765681.835572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765682.603725:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765682.859621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765683.627645:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765683.883557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765684.075582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765684.075592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765684.075596:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765684.075608:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1412 00000100:00080000:2.0:1775765684.075620:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765684.075626:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765684.075672:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861346304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765684.075701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765684.075704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765684.075705:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765684.075727:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1412 00000100:00080000:2.0:1775765684.075731:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765684.075734:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765684.075744:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765684.075753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861346432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765684.075761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765684.075764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765684.075765:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765684.075768:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1412 00000100:00080000:2.0:1775765684.075770:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765684.075773:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765684.075776:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765684.075783:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861346560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765684.075789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861346304:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775765684.075863:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765684.075906:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765684/real 1775765684] req@ffffa0e9d0711c00 x1862023861346304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765684.075924:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861346304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765684.075932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765684.075936:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765684.075940:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861346304:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765684.075945:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765684.075959:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765684.075977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765684/real 1775765684] req@ffffa0e9d0713b80 x1862023861346432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765684.075985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861346432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765684.075991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765684.075993:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765684.075996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346432:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765684.076021:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765684.076033:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765684.076039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765684/real 1775765684] req@ffffa0e9d0710000 x1862023861346560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765684.076046:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861346560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765684.076051:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765684.076053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765684.076054:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346560:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765684.651687:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765684.907629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765685.675770:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765685.931543:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765686.699679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765686.955692:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765687.723726:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765687.979612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765688.747709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765689.003614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765689.195636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765689.195644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765689.195648:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765689.195656:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1417 00000100:00080000:0.0:1775765689.195665:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765689.195671:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765689.195717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861346688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765689.195746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765689.195749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765689.195750:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765689.195753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1417 00000100:00080000:0.0:1775765689.195757:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765689.195760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765689.195768:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765689.195778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861346816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765689.195786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765689.195792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765689.195794:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765689.195795:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861346688:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765689.195801:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1417 00000100:00080000:0.0:1775765689.195812:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765689.195817:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765689.195829:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765689.195842:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861346944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765689.195894:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765689.195949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765689/real 1775765689] req@ffffa0e9d09de300 x1862023861346688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765689.195980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861346688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765689.195990:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765689.195994:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765689.195997:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861346688:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765689.196023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765689.196042:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765689.196051:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765689/real 1775765689] req@ffffa0e9d09dea00 x1862023861346816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765689.196059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861346816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765689.196064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765689.196067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765689.196069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346816:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765689.196072:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765689.196083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765689.196088:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765689/real 1775765689] req@ffffa0e9d09df480 x1862023861346944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765689.196096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861346944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765689.196110:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765689.196113:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765689.196115:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861346944:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765689.771572:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765690.027632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765690.795699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765691.051566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765691.819637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765692.075593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765692.843618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765693.099618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765693.867636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765694.123599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765694.315582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765694.315593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765694.315598:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765694.315632:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1422 00000100:00080000:0.0:1775765694.315644:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765694.315651:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765694.315692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861347072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765694.315721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765694.315724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765694.315725:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765694.315728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1422 00000100:00080000:0.0:1775765694.315731:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765694.315733:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765694.315743:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765694.315752:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861347200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765694.315760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765694.315763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765694.315764:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765694.315769:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861347072:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765694.315770:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1422 00000100:00080000:0.0:1775765694.315774:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765694.315781:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765694.315786:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765694.315799:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861347328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765694.315891:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765694.315915:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347328:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765694.891637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765695.147598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765695.915611:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765696.171637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765696.939623:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765697.195579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765697.387886:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765697.387945:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765697.387967:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765697.388012:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765697.388028:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:2.0:1775765697.388044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765694/real 1775765697] req@ffffa0e9d09dea00 x1862023861347200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765697.388072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861347200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765697.388087:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765697.388093:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765697.388132:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765697.388142:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765694/real 1775765697] req@ffffa0e9d09de300 x1862023861347328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765697.388157:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861347328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765697.388166:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765697.388169:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765697.388173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347328:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765697.388181:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765694/real 1775765697] req@ffffa0e9d09df480 x1862023861347072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765697.388194:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861347072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765697.388201:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765697.388205:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765697.388208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861347072:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775765697.963750:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765698.219625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765698.987645:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765699.243594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765699.435652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765699.435665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765699.435670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765699.435684:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1427 00000100:00080000:2.0:1775765699.435699:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765699.435708:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765699.435770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861347456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765699.435801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765699.435804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765699.435806:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765699.435809:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1427 00000100:00080000:2.0:1775765699.435813:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765699.435817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765699.435834:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861347456:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775765699.435847:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765699.435865:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861347584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765699.435881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765699.435885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765699.435887:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765699.435892:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1427 00000100:00080000:2.0:1775765699.435897:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765699.435900:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765699.435906:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765699.435919:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861347712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765699.435931:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765699.435990:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765699/real 1775765699] req@ffffa0e9d0711180 x1862023861347456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765699.436020:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861347456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765699.436033:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765699.436037:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765699.436041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861347456:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765699.436106:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765699.436133:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765699.436148:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765699/real 1775765699] req@ffffa0e9d0710380 x1862023861347584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765699.436160:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861347584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765699.436169:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765699.436172:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765699.436175:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347584:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765699.436181:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765699.436199:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765699.436219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765699/real 1775765699] req@ffffa0e9d0710000 x1862023861347712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765699.436231:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861347712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765699.436239:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765699.436242:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765699.436246:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347712:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765700.011743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765700.267623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765701.035641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765701.291604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765702.059657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765702.315679:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765703.083661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765703.339588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765704.107638:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765704.363648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765704.555658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765704.555666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765704.555670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765704.555678:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1433 00000100:00080000:0.0:1775765704.555687:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765704.555692:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765704.555725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861347840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765704.555753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765704.555755:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765704.555757:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765704.555759:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1433 00000100:00080000:0.0:1775765704.555762:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765704.555765:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765704.555774:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765704.555783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861347968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765704.555808:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861347840:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765704.555811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765704.555826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765704.555828:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765704.555835:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1433 00000100:00080000:0.0:1775765704.555839:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765704.555844:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765704.555848:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765704.555860:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861348096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765704.555900:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765704.555948:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765704/real 1775765704] req@ffffa0e9d09dc000 x1862023861347840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765704.555973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861347840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765704.555983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765704.555987:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765704.555991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861347840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765704.556016:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765704.556033:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765704.556042:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765704/real 1775765704] req@ffffa0e9d09dc380 x1862023861347968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765704.556063:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861347968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765704.556069:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765704.556072:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765704.556074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861347968:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765704.556078:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765704.556110:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765704.556117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765704/real 1775765704] req@ffffa0e9d09dce00 x1862023861348096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765704.556126:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861348096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765704.556131:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765704.556133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765704.556135:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348096:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765705.131635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765705.387590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765706.155717:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765706.411628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765707.179645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765707.435636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765708.203628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765708.459623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765709.227680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765709.483638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765709.675557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765709.675565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765709.675569:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765709.675577:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1438 00000100:00080000:0.0:1775765709.675587:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765709.675592:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765709.675639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861348224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765709.675676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765709.675680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765709.675682:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765709.675687:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1438 00000100:00080000:0.0:1775765709.675690:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765709.675694:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765709.675729:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765709.675742:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861348224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765709.675743:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861348352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765709.675759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765709.675762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765709.675763:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765709.675767:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1438 00000100:00080000:0.0:1775765709.675770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765709.675772:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765709.675777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765709.675794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861348480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765709.675810:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765709.675856:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765709/real 1775765709] req@ffffa0e9d09ded80 x1862023861348224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765709.675873:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861348224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765709.675886:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765709.675891:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765709.675896:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861348224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765709.675956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765709.675981:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765709.675992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765709/real 1775765709] req@ffffa0e9d09df100 x1862023861348352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765709.676020:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861348352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765709.676027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765709.676030:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765709.676032:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348352:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765709.676043:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765709.676060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765709.676068:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765709/real 1775765709] req@ffffa0e9d09dfb80 x1862023861348480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765709.676077:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861348480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765709.676083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765709.676085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765709.676087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348480:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765710.251789:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765710.507614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765711.275614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765711.531577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765712.299666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765712.555569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765713.323607:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765713.579643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765714.347618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765714.603618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765714.795691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765714.795712:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765714.795717:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765714.795727:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1443 00000100:00080000:2.0:1775765714.795739:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765714.795745:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765714.795796:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861348608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765714.795835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765714.795840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765714.795843:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765714.795850:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1443 00000100:00080000:2.0:1775765714.795856:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765714.795862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765714.795876:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765714.795897:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861348608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765714.795915:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861348736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765714.795982:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765714.796006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765714.796008:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765714.796015:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1443 00000100:00080000:2.0:1775765714.796021:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765714.796025:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765714.796048:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765714.796061:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765714.796067:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861348864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765714.796120:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765714/real 1775765714] req@ffffa0e9d0713b80 x1862023861348608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765714.796139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861348608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765714.796149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765714.796158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765714.796161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861348608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765714.796200:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765714.796223:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765714.796255:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765714/real 1775765714] req@ffffa0e9d0711c00 x1862023861348736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765714.796272:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861348736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765714.796283:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765714.796288:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765714.796292:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348736:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765714.796300:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765714.796329:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765714.796345:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765714/real 1775765714] req@ffffa0e9d0711f80 x1862023861348864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765714.796361:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861348864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765714.796378:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765714.796381:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765714.796384:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861348864:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765715.371768:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765715.627651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765716.395723:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765716.651614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765717.419671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765717.675583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765718.443700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765718.699570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765719.467756:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765719.723594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765719.915673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765719.915725:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765719.915732:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765719.915746:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1448 00000100:00080000:0.0:1775765719.915775:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765719.915783:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765719.915846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861348992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765719.915917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765719.915921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765719.915923:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765719.915927:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1448 00000100:00080000:0.0:1775765719.915932:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765719.915936:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765719.915948:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765719.915960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861349120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765719.915975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765719.915975:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861348992:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765719.915984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765719.915989:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765719.915995:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1448 00000100:00080000:0.0:1775765719.915998:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765719.916000:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765719.916005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765719.916017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861349248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765719.916071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765719.916117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765719/real 1775765719] req@ffffa0e9d09dd880 x1862023861348992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765719.916136:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861348992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765719.916147:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765719.916151:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765719.916154:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861348992:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765719.916174:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765719.916192:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765719.916200:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765719/real 1775765719] req@ffffa0e9d09dc700 x1862023861349120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765719.916208:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861349120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765719.916229:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765719.916233:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765719.916236:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349120:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765719.916243:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765719.916267:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765719.916279:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765719/real 1775765719] req@ffffa0e9d09df800 x1862023861349248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765719.916292:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861349248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765719.916298:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765719.916301:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765719.916303:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349248:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765720.491612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765720.747659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765721.515860:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765721.771623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765722.539769:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765722.795622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765723.563690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765723.819630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765724.587645:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765724.843635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765725.035594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765725.035603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765725.035607:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765725.035614:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1453 00000100:00080000:0.0:1775765725.035623:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765725.035629:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765725.035662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861349376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765725.035687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765725.035690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765725.035691:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765725.035694:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1453 00000100:00080000:0.0:1775765725.035696:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765725.035698:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765725.035705:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765725.035712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861349504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765725.035728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765725.035731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765725.035733:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765725.035736:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1453 00000100:00080000:0.0:1775765725.035738:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765725.035757:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765725.035763:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861349376:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775765725.035764:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765725.035782:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861349632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765725.035872:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765725.035935:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765725/real 1775765725] req@ffffa0e9d09dd180 x1862023861349376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765725.035970:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861349376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765725.035979:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765725.035983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765725.035986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861349376:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765725.035993:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765725.036012:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765725.036021:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765725/real 1775765725] req@ffffa0e9d09ddc00 x1862023861349504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765725.036029:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861349504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765725.036035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765725.036037:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765725.036039:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349504:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765725.036062:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765725.036073:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765725.036078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765725/real 1775765725] req@ffffa0e9d09dd500 x1862023861349632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765725.036086:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861349632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765725.036099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765725.036102:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765725.036104:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349632:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765725.611704:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765725.867616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765726.635688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765726.891642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765727.659796:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765727.915654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765728.683739:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765728.939538:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765729.707684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765729.963599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765730.155642:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765730.155659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765730.155666:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765730.155682:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1458 00000100:00080000:0.0:1775765730.155699:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765730.155728:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765730.155793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861349760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765730.155873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765730.155880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765730.155884:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765730.155892:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1458 00000100:00080000:0.0:1775765730.155900:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765730.155907:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765730.155924:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765730.155950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861349888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765730.155953:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861349760:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765730.155972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765730.156011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765730.156014:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765730.156021:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1458 00000100:00080000:0.0:1775765730.156028:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765730.156034:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765730.156044:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765730.156068:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765730.156074:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861350016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765730.156126:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765730/real 1775765730] req@ffffa0e9d09ddf80 x1862023861349760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765730.156160:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861349760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765730.156185:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765730.156193:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765730.156198:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861349760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765730.156229:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765730.156264:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765730.156281:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765730/real 1775765730] req@ffffa0e9d09dca80 x1862023861349888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765730.156317:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861349888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765730.156327:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765730.156331:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765730.156354:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861349888:192.168.202.104@tcp:38:kworker.0 00000100:00100000:2.0:1775765730.156368:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:2.0:1775765730.156396:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765730.156410:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765730/real 1775765730] req@ffffa0e9d09de680 x1862023861350016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765730.156434:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861350016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765730.156444:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765730.156447:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:2.0:1775765730.156451:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350016:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775765730.731627:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765730.987601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765731.755726:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765732.011655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765732.779601:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765733.035617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765733.803689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765734.059679:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765734.827788:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765735.083740:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765735.275682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765735.275724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765735.275730:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765735.275774:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1463 00000100:00080000:2.0:1775765735.275790:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765735.275799:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765735.275864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861350144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765735.275910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765735.275916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765735.275919:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765735.275925:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1463 00000100:00080000:2.0:1775765735.275930:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765735.275937:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765735.275953:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765735.275986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861350144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765735.275987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861350272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765735.276039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765735.276046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765735.276050:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765735.276059:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1463 00000100:00080000:2.0:1775765735.276069:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765735.276077:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765735.276104:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765735.276130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861350400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765735.276157:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765735.276251:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765735/real 1775765735] req@ffffa0e9d0710700 x1862023861350144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765735.276279:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861350144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765735.276293:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765735.276298:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765735.276304:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861350144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765735.276335:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765735.276366:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765735.276381:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765735/real 1775765735] req@ffffa0e9d0712d80 x1862023861350272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765735.276436:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861350272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765735.276447:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765735.276451:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765735.276455:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765735.276462:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765735.276487:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765735.276500:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765735/real 1775765735] req@ffffa0e9d0712680 x1862023861350400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765735.276514:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861350400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765735.276523:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765735.276526:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765735.276568:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350400:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765735.851643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765736.107663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775765736.875771:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765737.131619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765737.899840:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765738.155642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765738.923782:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765739.179658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765739.947732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765740.203647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765740.395602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765740.395618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765740.395635:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765740.395651:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1468 00000100:00080000:2.0:1775765740.395668:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765740.395679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765740.395755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861350528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765740.395796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765740.395800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765740.395802:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765740.395806:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1468 00000100:00080000:2.0:1775765740.395810:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765740.395814:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765740.395826:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765740.395840:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861350656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765740.395849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765740.395860:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861350528:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765740.395873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765740.395875:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765740.395880:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1468 00000100:00080000:2.0:1775765740.395884:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765740.395888:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765740.395893:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765740.395906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861350784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765740.395946:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765740.395995:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765740/real 1775765740] req@ffffa0e9d0710a80 x1862023861350528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765740.396015:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861350528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765740.396032:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765740.396036:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765740.396040:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861350528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765740.396067:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765740.396089:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765740.396100:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765740/real 1775765740] req@ffffa0e9d0710e00 x1862023861350656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765740.396109:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861350656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765740.396116:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765740.396119:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765740.396122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350656:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765740.396126:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765740.396138:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765740.396144:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765740/real 1775765740] req@ffffa0e9d0712a00 x1862023861350784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765740.396153:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861350784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765740.396161:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765740.396165:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765740.396169:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861350784:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765740.971757:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765741.227666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765741.995672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765742.251596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765743.019756:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765743.275610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765744.043644:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765744.299743:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765745.067731:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765745.323715:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765745.515750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765745.515763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765745.515770:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765745.515784:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1474 00000100:00080000:0.0:1775765745.515803:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765745.515814:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765745.515883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861350912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765745.515965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765745.515973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765745.515977:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765745.515985:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1474 00000100:00080000:0.0:1775765745.515994:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765745.516001:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765745.516045:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765745.516065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861351040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765745.516076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765745.516079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765745.516081:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765745.516085:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1474 00000100:00080000:0.0:1775765745.516089:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765745.516103:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861350912:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765745.516121:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765745.516130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765745.516147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861351168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765745.516191:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765745.516252:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765745/real 1775765745] req@ffffa0e9d09de680 x1862023861350912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765745.516275:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861350912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765745.516286:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765745.516290:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765745.516295:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861350912:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765745.516301:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765745.516322:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765745.516332:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765745/real 1775765745] req@ffffa0e9d09dca80 x1862023861351040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765745.516352:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861351040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765745.516359:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765745.516362:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765745.516365:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765745.516392:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765745.516409:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765745.516417:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765745/real 1775765745] req@ffffa0e9d09ddf80 x1862023861351168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765745.516427:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861351168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765745.516433:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765745.516436:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765745.516439:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351168:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765746.091820:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765746.347724:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765747.115749:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765747.371631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765748.139696:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765748.395725:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765749.163787:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765749.419603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765750.187648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765750.443624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765750.635706:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765750.635718:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765750.635724:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765750.635742:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1479 00000100:00080000:2.0:1775765750.635757:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765750.635764:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765750.635816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861351296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765750.635872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765750.635876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765750.635879:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765750.635883:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1479 00000100:00080000:2.0:1775765750.635888:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765750.635894:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765750.635907:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765750.635919:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861351296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765750.635921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861351424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765750.635931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765750.635935:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765750.635937:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765750.635941:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1479 00000100:00080000:2.0:1775765750.635944:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765750.635948:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765750.635953:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765750.635963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861351552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765750.636008:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765750.636061:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765750/real 1775765750] req@ffffa0e9d0713480 x1862023861351296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765750.636079:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861351296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765750.636095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765750.636099:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765750.636102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861351296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765750.636127:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765750.636167:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765750.636180:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765750/real 1775765750] req@ffffa0e9d0711500 x1862023861351424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765750.636191:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861351424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765750.636198:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765750.636202:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765750.636205:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351424:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765750.636209:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765750.636223:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765750.636231:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765750/real 1775765750] req@ffffa0e9d0712300 x1862023861351552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765750.636241:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861351552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765750.636248:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765750.636263:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765750.636266:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351552:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765751.211631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765751.467587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765752.235716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765752.491582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765753.259804:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765753.515623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765754.283644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765754.539592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765755.307707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765755.563629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765755.755683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765755.755695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765755.755701:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765755.755712:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1484 00000100:00080000:1.0:1775765755.755725:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765755.755732:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765755.755778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861351680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765755.755817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765755.755839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765755.755842:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765755.755847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1484 00000100:00080000:1.0:1775765755.755862:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765755.755868:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765755.755882:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765755.755900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861351808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765755.755911:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861351680:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765755.755927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765755.755932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765755.755934:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765755.755940:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1484 00000100:00080000:1.0:1775765755.755944:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765755.755972:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765755.755981:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765755.756018:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861351936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765755.756057:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765755.756109:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765755/real 1775765755] req@ffffa0e9c8ded180 x1862023861351680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765755.756139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861351680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765755.756153:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765755.756158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765755.756162:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861351680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765755.756194:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765755.756233:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765755.756256:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765755/real 1775765755] req@ffffa0e9c8dec000 x1862023861351808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765755.756279:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861351808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765755.756323:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765755.756352:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765755.756357:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351808:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765755.756365:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765755.756393:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765755.756406:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765755/real 1775765755] req@ffffa0e9c8dee300 x1862023861351936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765755.756419:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861351936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765755.756428:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765755.756432:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765755.756435:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861351936:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765756.331741:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765756.587591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765757.355717:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765757.611606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765758.379722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765758.635658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765759.403705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765759.659556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765760.427666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765760.683662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765760.875659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765760.875669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765760.875675:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765760.875685:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1489 00000100:00080000:0.0:1775765760.875697:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765760.875704:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765760.875751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861352064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765760.875778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765760.875782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765760.875803:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765760.875808:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1489 00000100:00080000:0.0:1775765760.875811:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765760.875815:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861352064:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765760.875819:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765760.875833:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765760.875851:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861352192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765760.875864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765760.875869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765760.875870:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765760.875875:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1489 00000100:00080000:0.0:1775765760.875890:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765760.875895:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765760.875900:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765760.875911:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861352320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765760.875957:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765760.875995:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352320:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765761.451705:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765761.707628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765762.475790:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765762.731652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765763.499635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765763.755610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775765763.947971:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775765763.948073:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765763.948095:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765763.948146:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765763.948159:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775765763.948233:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765760/real 1775765763] req@ffffa0e9d09ddc00 x1862023861352192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765763.948262:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861352192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765763.948271:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765763.948275:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765763.948278:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765763.948284:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765760/real 1775765763] req@ffffa0e9d09dd180 x1862023861352320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765763.948292:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861352320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765763.948296:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765763.948316:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765763.948318:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352320:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765763.948324:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765760/real 1775765763] req@ffffa0e9d09dd500 x1862023861352064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765763.948333:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861352064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765763.948338:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765763.948340:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765763.948342:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861352064:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775765764.523623:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765764.779692:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765765.547691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765765.803630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765765.995613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765765.995626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765765.995632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765765.995646:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1494 00000100:00080000:1.0:1775765765.995660:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765765.995670:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765765.995715:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861352448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765765.995745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765765.995747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765765.995749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765765.995752:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1494 00000100:00080000:1.0:1775765765.995755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765765.995777:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765765.995789:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765765.995797:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861352448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765765.995802:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861352576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765765.995824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765765.995826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765765.995828:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765765.995831:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1494 00000100:00080000:1.0:1775765765.995834:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765765.995836:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765765.995838:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765765.995848:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861352704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765765.995925:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765765.995999:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765765/real 1775765765] req@ffffa0e9c8dec380 x1862023861352448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765765.996025:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861352448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765765.996047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765765.996053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765765.996058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861352448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765765.996094:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765765.996124:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765765.996141:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765765/real 1775765765] req@ffffa0e9c8deed80 x1862023861352576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765765.996155:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861352576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765765.996165:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765765.996169:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765765.996172:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352576:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765765.996179:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765765.996202:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765765.996215:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765765/real 1775765765] req@ffffa0e9c8ded500 x1862023861352704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765765.996229:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861352704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765765.996249:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765765.996253:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765765.996256:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352704:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765766.571642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765766.827653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765767.595657:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765767.851604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765768.619662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765768.875634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765769.643590:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765769.899578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765770.667670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765770.923553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765771.115625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765771.115639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765771.115647:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765771.115663:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1499 00000100:00080000:2.0:1775765771.115680:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765771.115691:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765771.115751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861352832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765771.115793:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765771.115820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765771.115823:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765771.115828:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1499 00000100:00080000:2.0:1775765771.115835:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765771.115843:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765771.115858:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765771.115875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861352960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765771.115891:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861352832:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765771.115897:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765771.115903:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765771.115906:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765771.115913:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1499 00000100:00080000:2.0:1775765771.115919:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765771.115925:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765771.115932:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765771.115950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861353088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765771.115964:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765771.116016:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765771/real 1775765771] req@ffffa0e9d0713100 x1862023861352832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765771.116038:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861352832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765771.116048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765771.116052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765771.116055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861352832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765771.116077:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765771.116106:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765771.116116:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765771/real 1775765771] req@ffffa0e9d0711880 x1862023861352960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765771.116125:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861352960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765771.116131:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765771.116133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765771.116135:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861352960:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765771.116139:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765771.116152:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765771.116174:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765771/real 1775765771] req@ffffa0e9d0712300 x1862023861353088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765771.116182:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861353088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765771.116187:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765771.116196:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765771.116198:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353088:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765771.691819:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765771.947582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765772.715610:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765772.971615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765773.739688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765773.995611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765774.763799:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765775.019599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765775.787611:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765776.043590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765776.235626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765776.235636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765776.235641:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765776.235652:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1504 00000100:00080000:0.0:1775765776.235664:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765776.235671:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765776.235715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861353216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765776.235748:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765776.235751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765776.235753:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765776.235756:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1504 00000100:00080000:0.0:1775765776.235758:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765776.235765:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765776.235776:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765776.235803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861353216:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765776.235812:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861353344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765776.235828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765776.235833:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765776.235835:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765776.235841:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1504 00000100:00080000:0.0:1775765776.235848:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765776.235854:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765776.235861:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765776.235875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861353472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765776.235883:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765776.235928:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765776/real 1775765776] req@ffffa0e9d09df800 x1862023861353216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765776.235943:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861353216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765776.235951:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765776.235997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765776.236001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861353216:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765776.236028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765776.236054:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765776.236065:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765776/real 1775765776] req@ffffa0e9d09dc700 x1862023861353344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765776.236077:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861353344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765776.236084:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765776.236087:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765776.236090:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353344:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765776.236095:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765776.236109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765776.236116:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765776/real 1775765776] req@ffffa0e9d09dd880 x1862023861353472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765776.236126:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861353472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765776.236133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765776.236135:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765776.236138:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353472:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765776.811650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765777.067562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765777.835682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765778.091577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765778.859663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765779.115587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765779.883666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765780.139597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765780.907742:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765781.163595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765781.355603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765781.355613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765781.355619:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765781.355630:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1509 00000100:00080000:1.0:1775765781.355643:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765781.355650:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765781.355697:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861353600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765781.355731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765781.355734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765781.355736:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765781.355739:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1509 00000100:00080000:1.0:1775765781.355741:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765781.355744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765781.355772:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765781.355790:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861353728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765781.355803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861353600:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765781.355810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765781.355816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765781.355819:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765781.355826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1509 00000100:00080000:1.0:1775765781.355832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765781.355838:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765781.355869:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765781.355883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861353856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765781.355889:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765781.355961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765781/real 1775765781] req@ffffa0e9c8def480 x1862023861353600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765781.355978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861353600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765781.355987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765781.355991:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765781.355996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861353600:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765781.356026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765781.356064:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765781.356079:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765781/real 1775765781] req@ffffa0e9c8dedc00 x1862023861353728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765781.356090:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861353728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765781.356096:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765781.356099:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765781.356102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353728:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765781.356107:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765781.356122:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765781.356129:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765781/real 1775765781] req@ffffa0e9c8def800 x1862023861353856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765781.356138:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861353856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765781.356143:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765781.356145:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765781.356147:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861353856:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765781.931641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765782.187646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765782.955817:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765783.211632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765783.979608:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765784.235651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765785.003610:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765785.259676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765786.027666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765786.283626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765786.475613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765786.475625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765786.475631:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765786.475643:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1514 00000100:00080000:0.0:1775765786.475684:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765786.475692:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765786.475741:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861353984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765786.475778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765786.475782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765786.475784:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765786.475788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1514 00000100:00080000:0.0:1775765786.475791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765786.475795:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765786.475806:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765786.475821:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861354112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765786.475830:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861353984:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765786.475836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765786.475841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765786.475842:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765786.475847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1514 00000100:00080000:0.0:1775765786.475849:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765786.475852:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765786.475856:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765786.475868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861354240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765786.475961:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765786.476015:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765786/real 1775765786] req@ffffa0e9d09dfb80 x1862023861353984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765786.476054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861353984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765786.476065:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765786.476069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765786.476074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861353984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765786.476101:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765786.476123:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765786.476134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765786/real 1775765786] req@ffffa0e9d09df100 x1862023861354112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765786.476146:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861354112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765786.476153:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765786.476157:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765786.476169:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354112:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765786.476175:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765786.476194:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765786.476203:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765786/real 1775765786] req@ffffa0e9d09ded80 x1862023861354240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765786.476214:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861354240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765786.476221:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765786.476224:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765786.476227:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354240:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765787.051723:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765787.307604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765788.075659:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765788.331624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765789.099658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765789.355612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765790.123689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765790.379580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765791.147606:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765791.403541:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765791.595686:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765791.595700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765791.595707:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765791.595721:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1520 00000100:00080000:1.0:1775765791.595737:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765791.595747:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765791.595806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861354368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765791.595846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765791.595850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765791.595852:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765791.595856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1520 00000100:00080000:1.0:1775765791.595885:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765791.595891:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765791.595900:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861354368:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775765791.595906:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765791.595925:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861354496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765791.595937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765791.595942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765791.595944:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765791.595949:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1520 00000100:00080000:1.0:1775765791.595952:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765791.595956:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765791.595960:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765791.595970:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861354624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765791.595996:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765791.596068:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765791/real 1775765791] req@ffffa0e9c8deea00 x1862023861354368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765791.596095:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861354368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765791.596107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765791.596112:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765791.596116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861354368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765791.596144:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765791.596167:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765791.596179:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765791/real 1775765791] req@ffffa0e9c8dee680 x1862023861354496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765791.596191:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861354496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765791.596199:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765791.596202:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765791.596206:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354496:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765791.596210:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765791.596253:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765791.596263:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765791/real 1775765791] req@ffffa0e9c8ded880 x1862023861354624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765791.596287:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861354624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765791.596295:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765791.596298:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765791.596301:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354624:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765792.171810:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765792.427636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765793.195709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765793.451567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765794.219657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765794.475642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765795.243781:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765795.499633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765796.267643:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765796.523608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765796.715566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765796.715577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765796.715583:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765796.715605:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1525 00000100:00080000:1.0:1775765796.715618:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765796.715625:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765796.715681:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861354752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765796.715734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765796.715737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765796.715743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765796.715747:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1525 00000100:00080000:1.0:1775765796.715750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765796.715753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765796.715764:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765796.715774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861354880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765796.715782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765796.715785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765796.715786:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765796.715789:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1525 00000100:00080000:1.0:1775765796.715792:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765796.715797:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861354752:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765796.715798:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765796.715803:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765796.715810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861355008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765796.715875:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765796.715937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765796/real 1775765796] req@ffffa0e9c8deca80 x1862023861354752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765796.715968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861354752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765796.715981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765796.715986:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765796.715991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861354752:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765796.716000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765796.716025:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765796.716039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765796/real 1775765796] req@ffffa0e9c8dedf80 x1862023861354880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765796.716048:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861354880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765796.716061:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765796.716064:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765796.716066:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861354880:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765796.716090:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765796.716106:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765796.716113:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765796/real 1775765796] req@ffffa0e9c8dece00 x1862023861355008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765796.716121:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861355008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765796.716126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765796.716128:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765796.716130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355008:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765797.291793:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765797.547575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765798.315749:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765798.571616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765799.339684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765799.595566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765800.363591:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765800.619563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765801.387831:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765801.643648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765801.835644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765801.835659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765801.835681:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765801.835718:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1530 00000100:00080000:1.0:1775765801.835730:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765801.835738:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765801.835794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861355136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765801.835837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765801.835840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765801.835843:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765801.835847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1530 00000100:00080000:1.0:1775765801.835850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765801.835854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765801.835863:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765801.835875:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861355264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765801.835885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765801.835889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765801.835890:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765801.835893:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1530 00000100:00080000:1.0:1775765801.835896:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765801.835899:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765801.835902:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765801.835912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861355392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765801.836004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861355136:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775765801.836183:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765801.836286:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765801/real 1775765801] req@ffffa0e9c8def100 x1862023861355136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765801.836319:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861355136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765801.836334:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765801.836341:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765801.836366:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861355136:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765801.836389:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765801.836424:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765801.836458:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765801/real 1775765801] req@ffffa0e9c8dec700 x1862023861355264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765801.836531:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861355264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765801.836543:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765801.836547:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765801.836551:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765801.836559:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765801.836585:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765801.836609:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765801/real 1775765801] req@ffffa0e9c8defb80 x1862023861355392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765801.836622:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861355392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765801.836659:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765801.836663:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765801.836667:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355392:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765802.411724:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765802.667572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765803.435588:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765803.691614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765804.459737:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765804.715659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765805.483745:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765805.739659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765806.507599:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765806.763632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765806.955639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765806.955652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765806.955659:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765806.955671:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1535 00000100:00080000:2.0:1775765806.955686:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765806.955719:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765806.955761:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861355520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765806.955790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765806.955794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765806.955795:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765806.955799:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1535 00000100:00080000:2.0:1775765806.955802:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765806.955805:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765806.955816:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765806.955824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861355648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765806.955832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765806.955835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765806.955836:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765806.955839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1535 00000100:00080000:2.0:1775765806.955841:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765806.955843:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765806.955845:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765806.955852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861355776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765806.955983:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861355520:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775765806.956122:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765806.956171:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765806/real 1775765806] req@ffffa0e9d0711500 x1862023861355520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765806.956189:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861355520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765806.956200:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765806.956204:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765806.956208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861355520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765806.956232:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765806.956256:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765806.956268:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765806/real 1775765806] req@ffffa0e9d0713480 x1862023861355648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765806.956280:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861355648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765806.956288:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765806.956291:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765806.956294:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355648:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765806.956300:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765806.956332:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765806.956340:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765806/real 1775765806] req@ffffa0e9d0712a00 x1862023861355776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765806.956372:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861355776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765806.956379:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765806.956382:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765806.956385:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861355776:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765807.531700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765807.787661:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765808.555782:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765808.811659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765809.579603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765809.835587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765810.603744:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765810.859622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765811.627675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765811.883633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765812.075556:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765812.075567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765812.075572:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765812.075583:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1540 00000100:00080000:2.0:1775765812.075595:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765812.075602:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765812.075650:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861355904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765812.075685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765812.075689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765812.075691:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765812.075695:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1540 00000100:00080000:2.0:1775765812.075700:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765812.075704:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765812.075716:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765812.075730:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861356032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765812.075743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765812.075746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765812.075749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765812.075753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1540 00000100:00080000:2.0:1775765812.075758:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765812.075769:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861355904:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765812.075783:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765812.075791:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765812.075805:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861356160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765812.075846:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765812.075889:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765812/real 1775765812] req@ffffa0e9d0710e00 x1862023861355904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765812.075904:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861355904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765812.075913:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765812.075917:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765812.075921:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861355904:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765812.075926:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765812.075940:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765812.075948:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765812/real 1775765812] req@ffffa0e9d0710a80 x1862023861356032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765812.075956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861356032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765812.075969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765812.075972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765812.075974:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765812.075998:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765812.076012:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765812.076018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765812/real 1775765812] req@ffffa0e9d0712680 x1862023861356160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765812.076045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861356160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765812.076051:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765812.076053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765812.076055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356160:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765812.651699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765812.907696:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765813.675661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765813.931612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765814.699720:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765814.955612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765815.723672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765815.979594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765816.747903:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765817.003663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765817.195668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765817.195686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765817.195695:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765817.195713:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1545 00000100:00080000:2.0:1775765817.195733:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765817.195746:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765817.195820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861356288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765817.195874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765817.195881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765817.195885:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765817.195893:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1545 00000100:00080000:2.0:1775765817.195900:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765817.195908:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765817.195927:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765817.195953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861356416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765817.196010:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861356288:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765817.196045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765817.196051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765817.196055:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765817.196064:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1545 00000100:00080000:2.0:1775765817.196073:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765817.196081:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765817.196091:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765817.196114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861356544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765817.196132:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765817.196208:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765817/real 1775765817] req@ffffa0e9d0712d80 x1862023861356288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765817.196244:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861356288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765817.196258:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765817.196264:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765817.196291:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861356288:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765817.196327:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765817.196369:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765817.196384:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765817/real 1775765817] req@ffffa0e9d0710700 x1862023861356416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765817.196398:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861356416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765817.196407:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765817.196411:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765817.196414:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356416:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765817.196421:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765817.196438:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765817.196447:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765817/real 1775765817] req@ffffa0e9d0711f80 x1862023861356544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765817.196458:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861356544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765817.196467:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765817.196469:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765817.196472:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356544:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765817.771794:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765818.027683:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765818.795694:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765819.051659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765819.819788:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765820.075657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765820.843851:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765821.099607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765821.867668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765822.123592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765822.315592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765822.315601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765822.315605:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765822.315612:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1550 00000100:00080000:2.0:1775765822.315621:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765822.315627:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765822.315660:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861356672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765822.315683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765822.315686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765822.315702:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765822.315705:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1550 00000100:00080000:2.0:1775765822.315709:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765822.315713:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765822.315720:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765822.315730:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861356800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765822.315735:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861356672:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765822.315742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765822.315745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765822.315746:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765822.315750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1550 00000100:00080000:2.0:1775765822.315752:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765822.315754:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765822.315758:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765822.315765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861356928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765822.315801:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765822.315840:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765822/real 1775765822] req@ffffa0e9d0711c00 x1862023861356672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765822.315870:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861356672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765822.315897:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765822.315903:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765822.315909:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861356672:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765822.315976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765822.315997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765822.316008:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765822/real 1775765822] req@ffffa0e9d0713b80 x1862023861356800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765822.316020:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861356800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765822.316028:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765822.316032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765822.316035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356800:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765822.316042:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765822.316059:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765822.316069:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765822/real 1775765822] req@ffffa0e9d0710000 x1862023861356928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765822.316096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861356928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765822.316104:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765822.316108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765822.316111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861356928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765822.891669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765823.147572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765823.915729:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765824.171694:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765824.939625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765825.195620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765825.963776:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765826.219561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765826.987688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765827.243616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765827.435638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765827.435649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765827.435655:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765827.435665:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1555 00000100:00080000:1.0:1775765827.435678:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765827.435684:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765827.435743:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861357056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765827.435771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765827.435774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765827.435776:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765827.435779:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1555 00000100:00080000:1.0:1775765827.435781:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765827.435783:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765827.435793:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765827.435802:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861357184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765827.435832:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765827.435835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765827.435836:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765827.435840:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1555 00000100:00080000:1.0:1775765827.435843:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765827.435845:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765827.435849:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765827.435858:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861357056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765827.435859:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861357312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765827.436018:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357184:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765827.436062:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357312:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765828.011649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765828.267613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765829.035830:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765829.291627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775765830.059688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765830.315580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765830.507818:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765830.507882:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765830.507903:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765830.507949:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765830.507960:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765830.508015:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765827/real 1775765830] req@ffffa0e9c8def100 x1862023861357312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765830.508048:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861357312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765830.508062:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765830.508069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765830.508075:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357312:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765830.508086:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765827/real 1775765830] req@ffffa0e9c8defb80 x1862023861357056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765830.508130:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861357056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765830.508140:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765830.508145:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765830.508148:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861357056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765830.508156:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765827/real 1775765830] req@ffffa0e9c8dec700 x1862023861357184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765830.508168:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861357184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765830.508177:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765830.508181:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765830.508185:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357184:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765831.083666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765831.339596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765832.107751:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765832.363639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765832.555606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765832.555616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765832.555621:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765832.555633:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1561 00000100:00080000:0.0:1775765832.555645:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765832.555653:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765832.555691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861357440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765832.555729:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765832.555751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765832.555753:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765832.555758:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1561 00000100:00080000:0.0:1775765832.555763:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765832.555768:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765832.555779:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765832.555802:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861357568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765832.555817:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765832.555823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765832.555841:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765832.555846:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1561 00000100:00080000:0.0:1775765832.555853:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765832.555853:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861357440:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765832.555866:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765832.555873:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765832.555889:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861357696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765832.555932:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765832.555973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765832/real 1775765832] req@ffffa0e9d09dce00 x1862023861357440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765832.556002:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861357440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765832.556014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765832.556019:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765832.556025:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861357440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765832.556033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765832.556060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765832.556084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765832/real 1775765832] req@ffffa0e9d09dc380 x1862023861357568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765832.556098:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861357568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765832.556107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765832.556110:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765832.556114:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357568:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765832.556149:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765832.556173:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765832.556186:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765832/real 1775765832] req@ffffa0e9d09dc000 x1862023861357696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765832.556200:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861357696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765832.556209:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765832.556213:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765832.556217:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357696:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775765833.131657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765833.387620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765834.155629:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765834.411653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765835.179853:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765835.435609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765836.203662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765836.459631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765837.227700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765837.483598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765837.675653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765837.675667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765837.675675:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765837.675689:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1566 00000100:00080000:1.0:1775765837.675706:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765837.675741:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765837.675827:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861357824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765837.675862:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765837.675868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765837.675870:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765837.675877:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1566 00000100:00080000:1.0:1775765837.675885:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765837.675891:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765837.675907:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765837.675909:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861357824:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765837.675930:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861357952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765837.675946:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765837.675952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765837.675955:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765837.675961:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1566 00000100:00080000:1.0:1775765837.675984:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765837.675991:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765837.675998:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765837.676000:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765837.676020:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861358080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765837.676072:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765837/real 1775765837] req@ffffa0e9c8dece00 x1862023861357824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765837.676104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861357824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765837.676125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765837.676129:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765837.676134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861357824:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765837.676162:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765837.676188:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765837.676200:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765837/real 1775765837] req@ffffa0e9c8dedf80 x1862023861357952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765837.676211:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861357952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765837.676218:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765837.676221:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765837.676224:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861357952:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765837.676228:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765837.676242:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765837.676249:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765837/real 1775765837] req@ffffa0e9c8deca80 x1862023861358080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765837.676258:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861358080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765837.676265:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765837.676267:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765837.676270:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765838.251734:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765838.507666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765839.275688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765839.531589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765840.299679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765840.555590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765841.323664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765841.579647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765842.347759:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765842.603609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765842.795652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765842.795661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765842.795665:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765842.795674:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1571 00000100:00080000:0.0:1775765842.795684:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765842.795690:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765842.795766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861358208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765842.795823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765842.795825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765842.795827:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765842.795829:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1571 00000100:00080000:0.0:1775765842.795832:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765842.795835:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765842.795845:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765842.795855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861358336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765842.795863:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765842.795866:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765842.795867:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765842.795872:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1571 00000100:00080000:0.0:1775765842.795877:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765842.795879:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861358208:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765842.795881:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765842.795904:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765842.795922:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861358464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765842.795977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765842.796044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765842/real 1775765842] req@ffffa0e9d09df480 x1862023861358208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765842.796073:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861358208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765842.796083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765842.796086:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765842.796090:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861358208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765842.796114:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765842.796143:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765842.796151:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765842/real 1775765842] req@ffffa0e9d09de300 x1862023861358336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765842.796159:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861358336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765842.796165:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765842.796168:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765842.796170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765842.796173:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765842.796188:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765842.796214:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765842/real 1775765842] req@ffffa0e9d09dea00 x1862023861358464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765842.796228:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861358464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765842.796238:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765842.796241:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765842.796245:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358464:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765843.371568:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765843.627602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765844.395827:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765844.651641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765845.419782:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765845.675630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765846.443639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765846.699616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765847.467669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765847.723582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765847.915653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765847.915663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765847.915667:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765847.915718:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1576 00000100:00080000:0.0:1775765847.915734:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765847.915759:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765847.915800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861358592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765847.915824:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765847.915827:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765847.915829:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765847.915832:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1576 00000100:00080000:0.0:1775765847.915836:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765847.915839:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765847.915848:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765847.915857:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861358720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765847.915865:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765847.915867:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765847.915869:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765847.915871:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1576 00000100:00100000:1.0:1775765847.915879:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861358592:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765847.915889:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765847.915893:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765847.915898:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765847.915907:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861358848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765847.915955:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765847.916001:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765847/real 1775765847] req@ffffa0e9d09dea00 x1862023861358592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765847.916017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861358592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765847.916025:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765847.916028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765847.916031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861358592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765847.916037:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765847.916053:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765847.916060:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765847/real 1775765847] req@ffffa0e9d09de300 x1862023861358720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765847.916081:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861358720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765847.916086:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765847.916089:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765847.916091:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358720:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765847.916109:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765847.916122:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765847.916128:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765847/real 1775765847] req@ffffa0e9d09df480 x1862023861358848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765847.916135:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861358848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765847.916140:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765847.916141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765847.916143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861358848:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765848.491718:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765848.747747:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765849.515734:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765849.771635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765850.539674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765850.795687:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765851.563805:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765851.819678:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765852.587667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765852.843600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765853.035602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765853.035611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765853.035615:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765853.035623:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1581 00000100:00080000:1.0:1775765853.035646:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765853.035668:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765853.035703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861358976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765853.035735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765853.035740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765853.035742:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765853.035746:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1581 00000100:00080000:1.0:1775765853.035749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765853.035753:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765853.035762:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765853.035772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861359104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765853.035777:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861358976:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765853.035786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765853.035789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765853.035790:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765853.035794:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1581 00000100:00080000:1.0:1775765853.035796:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765853.035798:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765853.035801:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765853.035809:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861359232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765853.035850:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765853.035889:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765853/real 1775765853] req@ffffa0e9c8ded880 x1862023861358976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.035903:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861358976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.035910:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.035914:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765853.035916:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861358976:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765853.035934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765853.035951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765853.035959:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765853/real 1775765853] req@ffffa0e9c8dee680 x1862023861359104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.035967:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861359104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.035972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.035975:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765853.035984:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359104:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765853.035988:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765853.036002:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765853.036009:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765853/real 1775765853] req@ffffa0e9c8deea00 x1862023861359232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.036017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861359232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.036022:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.036024:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765853.036026:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359232:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765853.611686:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765853.867683:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765854.635717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765854.891599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765855.659652:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765855.915667:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765856.683672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765856.939563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765857.707678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765857.963645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765858.155638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765858.155647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765858.155651:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765858.155658:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1586 00000100:00080000:2.0:1775765858.155668:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765858.155673:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765858.155730:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861359360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765858.155751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765858.155754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765858.155755:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765858.155758:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1586 00000100:00080000:2.0:1775765858.155761:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765858.155763:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765858.155771:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765858.155778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861359488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765858.155785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765858.155788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765858.155790:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765858.155792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1586 00000100:00080000:2.0:1775765858.155794:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765858.155796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765858.155799:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765858.155803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861359360:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765858.155809:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861359616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765858.155859:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765858.155890:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765858/real 1775765858] req@ffffa0e9d0710380 x1862023861359360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765858.155923:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861359360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765858.155933:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765858.155938:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765858.155942:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861359360:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765858.155950:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765858.155974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765858.155992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765858/real 1775765858] req@ffffa0e9d0711180 x1862023861359488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765858.156003:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861359488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765858.156012:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765858.156016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765858.156019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359488:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765858.156046:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765858.156082:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765858.156093:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765858/real 1775765858] req@ffffa0e9d0710000 x1862023861359616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765858.156106:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861359616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765858.156114:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765858.156117:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765858.156119:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359616:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765858.731628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765858.987615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765859.755711:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765860.011671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765860.779667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765861.035751:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765861.803624:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765862.059599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765862.827664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765863.083555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765863.275639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765863.275647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765863.275651:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765863.275659:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1591 00000100:00080000:0.0:1775765863.275668:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765863.275673:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765863.275718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861359744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765863.275742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765863.275744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765863.275745:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765863.275748:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1591 00000100:00080000:0.0:1775765863.275751:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765863.275754:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765863.275762:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765863.275770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861359872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765863.275778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765863.275797:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765863.275798:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765863.275803:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1591 00000100:00100000:1.0:1775765863.275804:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861359744:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765863.275805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765863.275811:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765863.275817:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765863.275826:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861360000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765863.275869:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765863.275912:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765863/real 1775765863] req@ffffa0e9d09dc000 x1862023861359744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765863.275929:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861359744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765863.275937:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765863.275941:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765863.275944:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861359744:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765863.275965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765863.275990:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765863.276003:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765863/real 1775765863] req@ffffa0e9d09dc380 x1862023861359872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765863.276018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861359872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765863.276029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765863.276032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765863.276035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861359872:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765863.276049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765863.276066:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765863.276075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765863/real 1775765863] req@ffffa0e9d09dce00 x1862023861360000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765863.276084:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861360000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765863.276089:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765863.276092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765863.276094:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360000:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765863.851720:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765864.107751:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765864.875604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765865.131599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765865.899674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765866.155625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765866.923710:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765867.179689:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765867.947646:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765868.203585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765868.395555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765868.395565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765868.395571:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765868.395605:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1596 00000100:00080000:0.0:1775765868.395618:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765868.395625:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765868.395666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861360128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765868.395697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765868.395700:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765868.395701:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765868.395704:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1596 00000100:00080000:0.0:1775765868.395707:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765868.395709:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765868.395719:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765868.395731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861360256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765868.395738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765868.395741:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765868.395743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765868.395746:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1596 00000100:00080000:0.0:1775765868.395749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765868.395751:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765868.395755:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765868.395764:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861360128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765868.395766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861360384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765868.395843:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765868.395878:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765868/real 1775765868] req@ffffa0e9d09ded80 x1862023861360128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765868.395892:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861360128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765868.395900:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765868.395904:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765868.395907:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861360128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765868.395912:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765868.395942:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765868.395950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765868/real 1775765868] req@ffffa0e9d09df100 x1862023861360256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765868.395958:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861360256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765868.395963:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765868.395966:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765868.395967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765868.395986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765868.395997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765868.396016:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765868/real 1775765868] req@ffffa0e9d09dfb80 x1862023861360384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765868.396024:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861360384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765868.396029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765868.396031:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765868.396033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360384:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765868.971666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765869.227631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765869.995682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765870.251590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765871.019623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765871.275583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765872.043681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765872.299566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765873.067643:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765873.323633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765873.515608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765873.515617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765873.515621:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765873.515628:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1602 00000100:00080000:0.0:1775765873.515637:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765873.515642:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765873.515675:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861360512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765873.515699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765873.515702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765873.515703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765873.515706:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1602 00000100:00080000:0.0:1775765873.515709:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765873.515711:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765873.515719:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765873.515728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861360640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765873.515743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765873.515744:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861360512:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765873.515751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765873.515752:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765873.515756:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1602 00000100:00080000:0.0:1775765873.515758:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765873.515761:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765873.515774:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765873.515785:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861360768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765873.515817:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765873.515856:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765873/real 1775765873] req@ffffa0e9d09dd880 x1862023861360512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765873.515872:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861360512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765873.515880:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765873.515883:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765873.515886:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861360512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765873.515907:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765873.515930:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765873.515943:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765873/real 1775765873] req@ffffa0e9d09dc700 x1862023861360640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765873.515956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861360640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765873.515963:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765873.515965:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765873.515969:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360640:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765873.515975:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765873.515996:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765873.516018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765873/real 1775765873] req@ffffa0e9d09df800 x1862023861360768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765873.516040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861360768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765873.516046:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765873.516048:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765873.516050:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861360768:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765874.091630:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765874.347585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765875.115654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765875.371609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765876.139667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765876.395645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765877.163669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765877.419633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765878.187736:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765878.443577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765878.635630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765878.635638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765878.635643:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765878.635651:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1607 00000100:00080000:0.0:1775765878.635661:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765878.635666:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765878.635698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861360896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765878.635725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765878.635728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765878.635729:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765878.635732:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1607 00000100:00080000:0.0:1775765878.635734:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765878.635737:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765878.635744:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765878.635753:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861361024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765878.635760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765878.635763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765878.635764:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765878.635767:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1607 00000100:00100000:1.0:1775765878.635774:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861360896:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765878.635785:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765878.635789:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765878.635794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765878.635804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861361152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765878.635832:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765878.635875:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765878/real 1775765878] req@ffffa0e9d09dd500 x1862023861360896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765878.635895:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861360896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765878.635906:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765878.635909:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765878.635912:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861360896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765878.635918:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765878.635936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765878.635944:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765878/real 1775765878] req@ffffa0e9d09dd180 x1862023861361024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765878.635980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861361024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765878.635985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765878.635988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765878.635990:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765878.636007:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765878.636022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765878.636029:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765878/real 1775765878] req@ffffa0e9d09ddc00 x1862023861361152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765878.636037:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861361152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765878.636042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765878.636044:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765878.636046:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361152:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765879.211904:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765879.467603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765880.235605:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765880.491589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765881.259663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765881.515625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765882.283654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765882.539580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765883.307641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765883.563708:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765883.755638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765883.755648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765883.755654:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765883.755664:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1612 00000100:00080000:3.0:1775765883.755677:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765883.755688:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765883.755731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861361280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765883.755756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765883.755758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765883.755760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765883.755763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1612 00000100:00080000:3.0:1775765883.755765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765883.755768:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765883.755777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765883.755786:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861361408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765883.755795:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765883.755797:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765883.755799:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765883.755802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1612 00000100:00080000:3.0:1775765883.755804:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765883.755809:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861361280:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765883.755809:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765883.755834:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765883.755844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861361536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765883.755885:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765883.755934:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765883/real 1775765883] req@ffffa0e9d0540a80 x1862023861361280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765883.755963:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861361280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765883.755972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765883.755976:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765883.755980:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861361280:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765883.755985:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765883.756005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765883.756018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765883/real 1775765883] req@ffffa0e9d0542d80 x1862023861361408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765883.756030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861361408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765883.756039:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765883.756043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765883.756047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361408:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765883.756076:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765883.756093:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765883.756101:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765883/real 1775765883] req@ffffa0e9d0541f80 x1862023861361536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765883.756109:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861361536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765883.756124:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765883.756126:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765883.756128:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361536:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765884.331679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765884.587587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765885.355645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765885.611618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765886.379683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765886.635573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765887.403662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765887.659629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765888.427645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765888.683576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765888.875655:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765888.875665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765888.875670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765888.875680:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1617 00000100:00080000:2.0:1775765888.875693:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765888.875700:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765888.875766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861361664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765888.875792:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765888.875795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765888.875797:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765888.875800:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1617 00000100:00080000:2.0:1775765888.875803:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765888.875806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765888.875815:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765888.875824:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861361792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765888.875832:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765888.875835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765888.875837:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765888.875841:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1617 00000100:00080000:2.0:1775765888.875843:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765888.875845:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765888.875848:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765888.875855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861361920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765888.875875:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861361664:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775765888.875941:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765888.875979:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765888/real 1775765888] req@ffffa0e9d0713b80 x1862023861361664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765888.875999:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861361664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765888.876010:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765888.876015:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765888.876019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861361664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765888.876027:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765888.876049:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765888.876059:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765888/real 1775765888] req@ffffa0e9d0711c00 x1862023861361792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765888.876067:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861361792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765888.876090:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765888.876093:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765888.876095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361792:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765888.876117:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765888.876132:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765888.876139:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765888/real 1775765888] req@ffffa0e9d0711f80 x1862023861361920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765888.876147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861361920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765888.876152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765888.876154:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765888.876156:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861361920:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765889.451592:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765889.707617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765890.475651:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765890.731644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765891.499667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765891.755574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765892.523616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765892.779646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765893.547663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765893.803618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765893.995599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765893.995608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765893.995614:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765893.995625:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1622 00000100:00080000:1.0:1775765893.995637:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765893.995643:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765893.995698:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861362048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765893.995745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765893.995750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765893.995752:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765893.995756:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1622 00000100:00080000:1.0:1775765893.995762:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765893.995767:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765893.995780:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765893.995797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861362176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765893.995811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765893.995817:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861362048:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765893.995817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765893.995820:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765893.995831:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1622 00000100:00080000:1.0:1775765893.995836:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765893.995843:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765893.995850:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765893.995870:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861362304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765893.995942:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765893.995969:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362304:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765894.571630:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765894.827573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765895.595617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765895.851578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765896.619675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765896.875593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775765897.067761:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775765897.067808:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765897.067826:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765897.067875:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775765897.067885:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765897.067930:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765893/real 1775765897] req@ffffa0e9c8dedc00 x1862023861362176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765897.067959:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861362176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765897.067973:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765897.067979:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765897.067984:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765897.067993:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765893/real 1775765897] req@ffffa0e9c8def480 x1862023861362304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765897.068006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861362304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765897.068014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765897.068017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765897.068037:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362304:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765897.068044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765893/real 1775765897] req@ffffa0e9c8def800 x1862023861362048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765897.068057:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861362048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765897.068064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765897.068068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765897.068071:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861362048:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775765897.643647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765897.899547:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765898.667569:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765898.923584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765899.115633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765899.115642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765899.115646:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765899.115655:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1627 00000100:00080000:2.0:1775765899.115666:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765899.115672:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765899.115706:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861362432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765899.115736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765899.115740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765899.115741:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765899.115745:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1627 00000100:00080000:2.0:1775765899.115748:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765899.115751:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765899.115761:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765899.115808:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861362432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765899.115808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861362560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765899.115856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765899.115861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765899.115864:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765899.115869:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1627 00000100:00080000:2.0:1775765899.115876:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765899.115882:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765899.115890:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765899.115907:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765899.115911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861362688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765899.115977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765899/real 1775765899] req@ffffa0e9d0710700 x1862023861362432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765899.116003:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861362432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765899.116016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765899.116021:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765899.116027:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861362432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765899.116057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765899.116089:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765899.116107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765899/real 1775765899] req@ffffa0e9d0712d80 x1862023861362560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765899.116147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861362560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765899.116158:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765899.116163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765899.116167:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362560:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765899.116176:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765899.116232:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765899.116248:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765899/real 1775765899] req@ffffa0e9d0712680 x1862023861362688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765899.116265:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861362688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765899.116275:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765899.116280:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765899.116285:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362688:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765899.691682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765899.947579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765900.715650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765900.971581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765901.739627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765901.995570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765902.763636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765903.019584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765903.787750:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765904.043570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765904.235594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765904.235604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765904.235610:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765904.235620:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1632 00000100:00080000:0.0:1775765904.235632:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765904.235638:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765904.235683:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861362816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765904.235730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765904.235733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765904.235735:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765904.235738:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1632 00000100:00080000:0.0:1775765904.235741:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765904.235744:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765904.235754:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765904.235767:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861362944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765904.235776:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765904.235782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765904.235785:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765904.235786:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861362816:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765904.235792:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1632 00000100:00080000:0.0:1775765904.235801:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765904.235806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765904.235812:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765904.235829:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861363072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765904.235860:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765904.235905:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765904/real 1775765904] req@ffffa0e9d09ddf80 x1862023861362816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765904.235925:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861362816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765904.235934:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765904.235938:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765904.235941:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861362816:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765904.235961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765904.235976:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765904.235983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765904/real 1775765904] req@ffffa0e9d09dca80 x1862023861362944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765904.235992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861362944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765904.235997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765904.236000:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765904.236002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861362944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765904.236006:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765904.236021:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765904.236049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765904/real 1775765904] req@ffffa0e9d09de680 x1862023861363072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765904.236062:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861363072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765904.236071:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765904.236074:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765904.236076:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363072:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765904.811619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765905.067590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765905.835690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765906.091613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765906.859688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765907.115650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765907.883625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765908.139549:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765908.907632:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765909.163564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765909.355561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765909.355572:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765909.355576:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765909.355599:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1637 00000100:00080000:2.0:1775765909.355610:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765909.355632:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765909.355662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861363200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765909.355694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765909.355697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765909.355699:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765909.355705:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1637 00000100:00080000:2.0:1775765909.355720:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765909.355725:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765909.355736:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765909.355747:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861363200:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775765909.355753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861363328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765909.355767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765909.355771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765909.355773:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765909.355789:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1637 00000100:00080000:2.0:1775765909.355793:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765909.355796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765909.355800:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765909.355811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861363456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765909.355819:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765909.355864:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765909/real 1775765909] req@ffffa0e9d0710a80 x1862023861363200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765909.355879:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861363200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765909.355889:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765909.355894:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765909.355898:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861363200:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765909.355932:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765909.355955:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765909.355974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765909/real 1775765909] req@ffffa0e9d0710e00 x1862023861363328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765909.355995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861363328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765909.356001:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765909.356003:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765909.356005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363328:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765909.356010:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765909.356025:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765909.356032:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765909/real 1775765909] req@ffffa0e9d0712a00 x1862023861363456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765909.356040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861363456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765909.356045:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765909.356047:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765909.356049:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363456:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765909.931654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765910.187626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765910.955610:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765911.211651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765911.979639:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765912.235604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765913.003679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765913.259663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765914.027646:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765914.283634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765914.475611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765914.475623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765914.475629:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765914.475641:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1642 00000100:00080000:2.0:1775765914.475654:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765914.475662:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765914.475719:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861363584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765914.475746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765914.475749:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765914.475751:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765914.475754:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1642 00000100:00080000:2.0:1775765914.475756:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765914.475759:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765914.475768:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765914.475776:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861363712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765914.475785:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765914.475788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765914.475789:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765914.475792:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1642 00000100:00080000:2.0:1775765914.475794:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765914.475811:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861363584:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765914.475817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765914.475831:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765914.475844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861363840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765914.475899:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765914.475943:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765914/real 1775765914] req@ffffa0e9d0713480 x1862023861363584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765914.475973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861363584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765914.475982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765914.475985:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765914.475989:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861363584:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765914.475995:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765914.476026:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765914.476035:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765914/real 1775765914] req@ffffa0e9d0711500 x1862023861363712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765914.476060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861363712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765914.476067:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765914.476070:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765914.476072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363712:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765914.476110:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765914.476126:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765914.476133:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765914/real 1775765914] req@ffffa0e9d0712300 x1862023861363840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765914.476142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861363840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765914.476156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765914.476159:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765914.476161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861363840:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765915.051664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765915.307569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765916.075661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765916.331626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765917.099660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765917.355631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765918.123690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765918.379680:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765919.147699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765919.403681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765919.595652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765919.595665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765919.595688:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765919.595698:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1648 00000100:00080000:0.0:1775765919.595730:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765919.595738:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765919.595792:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861363968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765919.595850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765919.595857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765919.595860:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765919.595867:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1648 00000100:00080000:0.0:1775765919.595873:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765919.595879:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765919.595891:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765919.595907:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861364096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765919.595935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765919.595938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765919.595940:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765919.595945:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1648 00000100:00080000:0.0:1775765919.595948:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765919.595952:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765919.595957:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765919.595967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861364224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765919.595972:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861363968:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775765919.596068:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765919.596117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765919/real 1775765919] req@ffffa0e9d09de680 x1862023861363968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765919.596142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861363968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765919.596153:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765919.596157:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765919.596161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861363968:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765919.596168:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765919.596190:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765919.596201:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765919/real 1775765919] req@ffffa0e9d09dca80 x1862023861364096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765919.596211:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861364096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765919.596232:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765919.596235:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765919.596238:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364096:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765919.596260:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364224:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765919.596278:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765919.596287:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765919/real 1775765919] req@ffffa0e9d09ddf80 x1862023861364224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765919.596296:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861364224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765919.596302:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765919.596304:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765919.596307:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364224:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765920.171603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765920.427652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765921.195687:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765921.451550:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765922.219698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765922.475578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765923.243588:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765923.499551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765924.267637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765924.523583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765924.715614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765924.715625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765924.715630:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765924.715640:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1653 00000100:00080000:2.0:1775765924.715652:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765924.715659:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765924.715697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861364352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765924.715733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765924.715739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765924.715742:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765924.715748:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1653 00000100:00080000:2.0:1775765924.715776:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765924.715782:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765924.715789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861364352:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775765924.715797:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765924.715818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861364480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765924.715832:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765924.715836:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765924.715838:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765924.715841:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1653 00000100:00080000:2.0:1775765924.715846:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765924.715850:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765924.715854:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765924.715861:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775765924.715868:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861364608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765924.715907:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765924/real 1775765924] req@ffffa0e9d0711880 x1862023861364352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765924.715927:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861364352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765924.715977:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765924.715982:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765924.715986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861364352:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765924.716012:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765924.716071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765924.716096:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765924/real 1775765924] req@ffffa0e9d0713100 x1862023861364480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765924.716122:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861364480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765924.716138:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765924.716174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765924.716180:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364480:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765924.716189:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765924.716236:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765924.716253:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765924/real 1775765924] req@ffffa0e9d0712300 x1862023861364608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765924.716269:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861364608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765924.716279:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765924.716283:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765924.716287:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364608:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765925.291663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765925.547588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765926.315603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765926.571598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765927.339813:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765927.595582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765928.363635:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765928.619610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765929.387593:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765929.643635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765929.835597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765929.835617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765929.835621:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765929.835629:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1658 00000100:00080000:2.0:1775765929.835639:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765929.835644:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765929.835692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861364736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765929.835716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765929.835719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765929.835720:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765929.835723:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1658 00000100:00080000:2.0:1775765929.835726:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765929.835728:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765929.835736:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765929.835745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861364864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765929.835753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765929.835756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775765929.835759:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861364736:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765929.835759:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765929.835765:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1658 00000100:00080000:2.0:1775765929.835767:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765929.835770:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765929.835774:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765929.835784:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861364992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765929.835826:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765929.835873:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765929/real 1775765929] req@ffffa0e9d0711500 x1862023861364736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765929.835893:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861364736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765929.835931:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765929.835936:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765929.835939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861364736:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765929.835966:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765929.835997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765929.836009:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765929/real 1775765929] req@ffffa0e9d0713480 x1862023861364864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765929.836021:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861364864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765929.836030:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765929.836033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765929.836036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364864:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765929.836042:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765929.836071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765929.836080:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765929/real 1775765929] req@ffffa0e9d0712a00 x1862023861364992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765929.836089:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861364992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765929.836094:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765929.836096:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765929.836098:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861364992:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765930.411626:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765930.667588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765931.435616:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765931.691639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765932.459621:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765932.715579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765933.483724:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765933.739637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765934.507661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765934.763562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765934.955621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765934.955640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765934.955649:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765934.955668:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1663 00000100:00080000:2.0:1775765934.955701:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765934.955713:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765934.955798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861365120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765934.955839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765934.955845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765934.955863:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765934.955870:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1663 00000100:00080000:2.0:1775765934.955875:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765934.955880:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765934.955912:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765934.955933:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861365248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765934.955951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765934.955957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765934.955961:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765934.955962:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861365120:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765934.955973:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1663 00000100:00080000:2.0:1775765934.955978:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765934.955990:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765934.956026:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765934.956061:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861365376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765934.956092:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765934.956153:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765934/real 1775765934] req@ffffa0e9d0710e00 x1862023861365120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765934.956174:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861365120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765934.956186:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765934.956191:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765934.956195:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861365120:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765934.956224:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765934.956257:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765934.956278:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765934/real 1775765934] req@ffffa0e9d0710a80 x1862023861365248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765934.956290:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861365248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765934.956297:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765934.956300:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765934.956303:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365248:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765934.956308:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765934.956323:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765934.956331:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765934/real 1775765934] req@ffffa0e9d0712680 x1862023861365376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765934.956340:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861365376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765934.956346:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765934.956349:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765934.956351:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365376:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765935.531615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765935.787668:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765936.555698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765936.811584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765937.579650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765937.835622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765938.603604:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765938.859661:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765939.627684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765939.883627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765940.075581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765940.075592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765940.075598:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765940.075610:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1668 00000100:00080000:2.0:1775765940.075623:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765940.075630:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765940.075688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861365504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765940.075729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765940.075733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765940.075745:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765940.075751:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1668 00000100:00080000:2.0:1775765940.075755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765940.075759:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765940.075787:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765940.075815:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861365632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765940.075825:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861365504:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765940.075865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765940.075868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765940.075870:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765940.075874:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1668 00000100:00080000:2.0:1775765940.075878:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765940.075881:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765940.075887:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765940.075897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861365760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765940.075936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765940.076011:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765940/real 1775765940] req@ffffa0e9d0712d80 x1862023861365504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765940.076028:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861365504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765940.076048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765940.076052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765940.076055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861365504:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765940.076078:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765940.076097:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765940.076105:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765940/real 1775765940] req@ffffa0e9d0710700 x1862023861365632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765940.076114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861365632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765940.076119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765940.076121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765940.076123:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365632:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765940.076127:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765940.076137:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765940.076142:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765940/real 1775765940] req@ffffa0e9d0711f80 x1862023861365760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765940.076150:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861365760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765940.076165:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765940.076167:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765940.076170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861365760:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765940.651660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765940.907551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765941.675602:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765941.931611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765942.699733:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765942.955536:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765943.723651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765943.979577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765944.747707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765945.003584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765945.195672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765945.195706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765945.195710:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765945.195720:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1673 00000100:00080000:0.0:1775765945.195732:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765945.195739:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765945.195804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861365888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765945.195834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765945.195837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765945.195838:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765945.195841:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1673 00000100:00080000:0.0:1775765945.195843:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765945.195846:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765945.195857:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765945.195866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861366016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765945.195873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765945.195876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765945.195879:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765945.195882:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1673 00000100:00080000:0.0:1775765945.195883:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765945.195885:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765945.195889:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765945.195899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861366144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765945.195902:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861365888:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775765945.195982:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765945.196025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765945/real 1775765945] req@ffffa0e9d09ddc00 x1862023861365888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765945.196042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861365888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765945.196050:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765945.196053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765945.196056:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861365888:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765945.196063:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765945.196080:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765945.196092:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765945/real 1775765945] req@ffffa0e9d09dd180 x1862023861366016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765945.196104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861366016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765945.196133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765945.196138:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765945.196140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366016:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765945.196161:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765945.196177:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765945.196185:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765945/real 1775765945] req@ffffa0e9d09dd500 x1862023861366144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765945.196193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861366144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765945.196198:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765945.196200:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765945.196202:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366144:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765945.771643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765946.027589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765946.795771:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765947.051546:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765947.819629:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765948.075592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765948.843698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765949.099683:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765949.867729:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765950.123630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765950.315595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765950.315606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765950.315611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765950.315622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1678 00000100:00080000:0.0:1775765950.315635:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765950.315641:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765950.315700:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861366272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765950.315751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765950.315754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765950.315759:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765950.315762:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1678 00000100:00080000:0.0:1775765950.315764:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765950.315768:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765950.315777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765950.315798:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861366400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765950.315807:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765950.315809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765950.315811:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765950.315816:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861366272:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765950.315818:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1678 00000100:00080000:0.0:1775765950.315824:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765950.315827:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765950.315851:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765950.315861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861366528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765950.315898:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765950.315960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765950/real 1775765950] req@ffffa0e9d09df800 x1862023861366272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765950.315978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861366272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765950.315999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765950.316002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765950.316005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861366272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765950.316028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765950.316047:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765950.316056:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765950/real 1775765950] req@ffffa0e9d09dc700 x1862023861366400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765950.316064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861366400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765950.316070:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765950.316072:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765950.316074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366400:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765950.316077:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765950.316087:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765950.316093:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765950/real 1775765950] req@ffffa0e9d09dd880 x1862023861366528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765950.316100:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861366528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765950.316106:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765950.316108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765950.316109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366528:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765950.891654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765951.147587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765951.915712:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765952.171611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765952.939762:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765953.195617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765953.963609:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765954.219577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765954.987643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765955.243615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775765955.435721:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765955.435735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765955.435742:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765955.435756:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1683 00000100:00080000:0.0:1775765955.435772:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765955.435781:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765955.435842:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861366656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765955.435893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765955.435917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765955.435921:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765955.435928:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1683 00000100:00080000:0.0:1775765955.435936:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765955.435944:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765955.435962:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765955.435988:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861366784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765955.436011:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861366656:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775765955.436048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765955.436054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765955.436058:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765955.436065:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1683 00000100:00080000:0.0:1775765955.436107:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775765955.436115:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775765955.436117:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775765955.436128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765955.436147:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861366912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765955.436172:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765955/real 1775765955] req@ffffa0e9d09dfb80 x1862023861366656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765955.436193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861366656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765955.436203:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765955.436208:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765955.436212:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861366656:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775765955.436242:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765955.436272:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765955.436287:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765955/real 1775765955] req@ffffa0e9d09df100 x1862023861366784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765955.436318:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861366784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765955.436325:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765955.436328:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765955.436330:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366784:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775765955.436336:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775765955.436370:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775765955.436384:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765955/real 1775765955] req@ffffa0e9d09ded80 x1862023861366912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765955.436398:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861366912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765955.436407:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765955.436411:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775765955.436415:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861366912:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765956.011708:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765956.267569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765957.035695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765957.291671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765958.059709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765958.315675:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765959.083823:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765959.339599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765960.107680:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765960.363619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765960.555649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765960.555658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765960.555663:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765960.555671:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1689 00000100:00080000:1.0:1775765960.555681:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765960.555687:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765960.555740:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861367040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765960.555769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765960.555773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765960.555776:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765960.555781:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1689 00000100:00080000:1.0:1775765960.555785:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765960.555807:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765960.555818:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765960.555850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861367168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765960.555851:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861367040:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765960.555878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765960.555880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765960.555882:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765960.555886:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1689 00000100:00080000:1.0:1775765960.555889:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765960.555893:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765960.555898:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765960.555909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861367296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765960.556007:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367168:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765960.556049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367296:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765961.131678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765961.387613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765962.155630:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765962.411548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765963.179691:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765963.435601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775765963.627744:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775765963.627796:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765963.627815:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765963.627860:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775765963.627870:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775765963.627915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765960/real 1775765963] req@ffffa0e9c8deed80 x1862023861367168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765963.627944:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861367168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765963.627955:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765963.627960:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765963.627964:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367168:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765963.627973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765960/real 1775765963] req@ffffa0e9c8dec380 x1862023861367296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765963.627985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861367296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765963.627992:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765963.627996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765963.628009:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367296:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765963.628016:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765960/real 1775765963] req@ffffa0e9c8ded500 x1862023861367040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765963.628030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861367040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765963.628037:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765963.628041:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765963.628044:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861367040:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775765964.203654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765964.459547:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765965.227649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765965.483612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775765965.675612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765965.675620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765965.675624:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765965.675632:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1694 00000100:00080000:1.0:1775765965.675641:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765965.675647:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775765965.675676:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861367424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765965.675721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765965.675725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765965.675727:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765965.675732:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1694 00000100:00080000:1.0:1775765965.675736:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765965.675741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765965.675751:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765965.675766:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861367552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765965.675779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861367424:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775765965.675780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765965.675789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765965.675792:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765965.675797:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1694 00000100:00080000:1.0:1775765965.675801:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765965.675805:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775765965.675810:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765965.675823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861367680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765965.675856:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765965.675898:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765965/real 1775765965] req@ffffa0e9c8dee300 x1862023861367424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765965.675920:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861367424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765965.675931:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765965.675936:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765965.675940:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861367424:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765965.675967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765965.675993:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765965.676006:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765965/real 1775765965] req@ffffa0e9c8dec000 x1862023861367552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765965.676035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861367552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765965.676045:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765965.676049:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765965.676052:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367552:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765965.676059:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765965.676082:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765965.676095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765965/real 1775765965] req@ffffa0e9c8ded180 x1862023861367680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765965.676108:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861367680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765965.676117:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765965.676120:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765965.676123:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367680:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765966.251645:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765966.507567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765967.275669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765967.531588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765968.299696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765968.555601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765969.323617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765969.579580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765970.347643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765970.603613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765970.795674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765970.795684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765970.795690:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765970.795701:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1699 00000100:00080000:3.0:1775765970.795714:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765970.795837:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765970.795875:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861367808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765970.795904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765970.795907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765970.795909:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765970.795914:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1699 00000100:00080000:3.0:1775765970.795922:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765970.795925:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765970.795933:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765970.795941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861367936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765970.795956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765970.795973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765970.795976:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765970.795977:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861367808:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765970.795981:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1699 00000100:00080000:3.0:1775765970.795986:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765970.795989:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765970.795994:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765970.796006:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861368064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765970.796038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765970.796073:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765970/real 1775765970] req@ffffa0e9d0541180 x1862023861367808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765970.796086:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861367808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765970.796105:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765970.796108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765970.796111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861367808:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765970.796116:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765970.796132:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765970.796141:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765970/real 1775765970] req@ffffa0e9d0543480 x1862023861367936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765970.796149:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861367936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765970.796154:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765970.796156:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765970.796159:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861367936:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765970.796175:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765970.796186:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765970.796191:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765970/real 1775765970] req@ffffa0e9d0543100 x1862023861368064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765970.796199:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861368064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765970.796204:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765970.796206:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765970.796207:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368064:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765971.371629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765971.627590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765972.395683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765972.651614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765973.419634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765973.675560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765974.443614:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765974.699571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765975.467598:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765975.723574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765975.915560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765975.915568:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765975.915572:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765975.915581:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1704 00000100:00080000:3.0:1775765975.915591:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765975.915597:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765975.915630:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861368192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765975.915650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765975.915652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765975.915654:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765975.915657:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1704 00000100:00080000:3.0:1775765975.915659:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765975.915663:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775765975.915677:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861368192:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775765975.915684:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765975.915698:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861368320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765975.915707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765975.915710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765975.915712:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765975.915714:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1704 00000100:00080000:3.0:1775765975.915718:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765975.915720:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765975.915724:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765975.915733:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861368448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765975.915736:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765975.915769:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765975/real 1775765975] req@ffffa0e9d0543100 x1862023861368192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765975.915784:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861368192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765975.915791:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765975.915793:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765975.915796:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861368192:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765975.915813:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765975.915827:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765975.915835:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765975/real 1775765975] req@ffffa0e9d0543480 x1862023861368320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765975.915843:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861368320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765975.915849:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765975.915859:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765975.915861:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368320:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765975.915865:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765975.915876:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765975.915882:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765975/real 1775765975] req@ffffa0e9d0541180 x1862023861368448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765975.915897:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861368448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765975.915903:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765975.915905:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765975.915907:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368448:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765976.491644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765976.747596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765977.515615:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765977.771558:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765978.539631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765978.795655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765979.563652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765979.819603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765980.587628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765980.843591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765981.035557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765981.035566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765981.035570:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765981.035578:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1709 00000100:00080000:3.0:1775765981.035588:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765981.035593:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765981.035627:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861368576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765981.035651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765981.035654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765981.035669:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765981.035674:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1709 00000100:00080000:3.0:1775765981.035678:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765981.035681:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765981.035690:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765981.035699:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861368576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775765981.035703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861368704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765981.035714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765981.035717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765981.035718:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765981.035721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1709 00000100:00080000:3.0:1775765981.035724:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765981.035726:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765981.035729:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765981.035738:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861368832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765981.035763:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765981.035797:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765981/real 1775765981] req@ffffa0e9d0541f80 x1862023861368576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765981.035811:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861368576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765981.035818:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765981.035821:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765981.035824:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861368576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765981.035852:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765981.035864:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765981.035872:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765981/real 1775765981] req@ffffa0e9d0542d80 x1862023861368704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765981.035880:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861368704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765981.035896:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765981.035898:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765981.035901:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368704:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765981.035905:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765981.035918:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765981.035924:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765981/real 1775765981] req@ffffa0e9d0540a80 x1862023861368832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765981.035932:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861368832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765981.035937:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765981.035939:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765981.035941:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861368832:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765981.611649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765981.867563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765982.635650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765982.891573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765983.659627:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765983.915566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765984.683636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765984.939584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765985.707648:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765985.963537:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765986.155608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765986.155617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765986.155620:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765986.155628:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1714 00000100:00080000:2.0:1775765986.155638:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765986.155685:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765986.155715:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861368960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765986.155739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765986.155741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765986.155743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765986.155745:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1714 00000100:00080000:2.0:1775765986.155748:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765986.155750:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765986.155758:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765986.155767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861369088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765986.155774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765986.155776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765986.155777:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765986.155779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1714 00000100:00080000:2.0:1775765986.155781:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765986.155783:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765986.155785:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765986.155791:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861369216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765986.155818:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861368960:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775765986.155873:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765986.155915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765986/real 1775765986] req@ffffa0e9d0711c00 x1862023861368960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765986.155929:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861368960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765986.155938:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765986.155942:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765986.155952:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861368960:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765986.155958:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765986.155974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765986.155982:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765986/real 1775765986] req@ffffa0e9d0713b80 x1862023861369088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765986.155990:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861369088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765986.155996:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765986.155998:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765986.156008:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765986.156012:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765986.156028:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765986.156034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765986/real 1775765986] req@ffffa0e9d0710000 x1862023861369216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765986.156042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861369216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765986.156047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765986.156049:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765986.156051:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369216:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765986.731654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765986.987625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765987.755613:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765988.011596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765988.779599:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765989.035573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765989.803728:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765990.059627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765990.827584:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765991.083585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775765991.275547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765991.275555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765991.275559:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765991.275567:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1719 00000100:00080000:2.0:1775765991.275576:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765991.275582:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775765991.275616:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861369344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765991.275640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765991.275642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765991.275644:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765991.275647:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1719 00000100:00080000:2.0:1775765991.275649:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765991.275651:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765991.275659:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765991.275668:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861369472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765991.275676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765991.275679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765991.275680:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765991.275683:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1719 00000100:00080000:2.0:1775765991.275685:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765991.275688:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861369344:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775765991.275704:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775765991.275709:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765991.275720:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861369600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765991.275747:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765991.275787:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765991/real 1775765991] req@ffffa0e9d0711180 x1862023861369344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765991.275801:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861369344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765991.275808:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765991.275812:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765991.275814:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861369344:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765991.275820:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765991.275834:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765991.275842:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765991/real 1775765991] req@ffffa0e9d0710380 x1862023861369472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765991.275849:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861369472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765991.275854:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765991.275856:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765991.275858:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369472:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765991.275877:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765991.275887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765991.275893:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765991/real 1775765991] req@ffffa0e9d0710000 x1862023861369600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765991.275900:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861369600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765991.275924:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765991.275927:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765991.275929:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369600:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775765991.851676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765992.107581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765992.875724:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765993.131568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775765993.899643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765994.155566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765994.923620:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765995.179652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765995.947641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765996.203629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775765996.395648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765996.395677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765996.395682:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765996.395693:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1724 00000100:00080000:3.0:1775765996.395724:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765996.395732:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775765996.395792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861369728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765996.395827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765996.395831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765996.395833:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765996.395838:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1724 00000100:00080000:3.0:1775765996.395841:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765996.395845:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765996.395858:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765996.395873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861369856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765996.395888:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861369728:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775765996.395890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765996.395902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765996.395905:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765996.395910:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1724 00000100:00080000:3.0:1775765996.395914:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765996.395917:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775765996.395923:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765996.395936:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861369984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765996.395988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765996.396069:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765996/real 1775765996] req@ffffa0e9d0540e00 x1862023861369728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765996.396115:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861369728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765996.396126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765996.396130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765996.396134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861369728:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775765996.396176:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765996.396210:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765996.396246:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765996/real 1775765996] req@ffffa0e9d0540700 x1862023861369856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765996.396292:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861369856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765996.396308:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765996.396327:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765996.396334:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369856:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775765996.396346:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775765996.396384:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775765996.396399:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765996/real 1775765996] req@ffffa0e9d0543800 x1862023861369984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765996.396428:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861369984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765996.396437:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765996.396441:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775765996.396445:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861369984:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775765996.971598:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765997.227659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765997.995736:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765998.251660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775765999.019662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765999.275590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766000.043591:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766000.299584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766001.067664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766001.323630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766001.515611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766001.515621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766001.515626:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766001.515636:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1730 00000100:00080000:3.0:1775766001.515663:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766001.515669:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766001.515709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861370112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766001.515738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766001.515741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766001.515743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766001.515747:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1730 00000100:00080000:3.0:1775766001.515750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766001.515753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766001.515762:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766001.515773:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861370240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766001.515782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766001.515786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766001.515788:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766001.515791:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1730 00000100:00080000:3.0:1775766001.515794:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766001.515796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766001.515807:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861370112:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775766001.515820:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766001.515835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861370368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766001.515888:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766001.515941:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766001/real 1775766001] req@ffffa0e9d0542300 x1862023861370112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766001.515959:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861370112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766001.515968:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766001.515980:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766001.515984:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861370112:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766001.515992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766001.516014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766001.516025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766001/real 1775766001] req@ffffa0e9d0540380 x1862023861370240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766001.516036:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861370240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766001.516043:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766001.516045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766001.516049:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370240:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766001.516069:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766001.516085:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766001.516094:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766001/real 1775766001] req@ffffa0e9d0541c00 x1862023861370368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766001.516104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861370368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766001.516112:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766001.516115:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766001.516118:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370368:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766002.091650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766002.347615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766003.115618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766003.371553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766004.139678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766004.395588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766005.163658:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766005.419609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766006.187639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766006.443566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766006.635612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766006.635622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766006.635627:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766006.635637:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1735 00000100:00080000:3.0:1775766006.635650:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766006.635657:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766006.635716:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861370496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766006.635745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766006.635748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766006.635749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766006.635752:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1735 00000100:00080000:3.0:1775766006.635754:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766006.635768:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766006.635778:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766006.635788:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861370624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766006.635796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766006.635800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766006.635821:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766006.635836:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1735 00000100:00080000:3.0:1775766006.635839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766006.635842:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766006.635846:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766006.635856:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861370752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766006.635862:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861370496:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766006.635934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766006.635971:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766006/real 1775766006] req@ffffa0e9d0543b80 x1862023861370496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766006.635986:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861370496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766006.635995:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766006.635998:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766006.636001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861370496:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766006.636007:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766006.636022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766006.636029:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766006/real 1775766006] req@ffffa0e9d0540000 x1862023861370624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766006.636037:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861370624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766006.636043:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766006.636045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766006.636047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370624:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766006.636068:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766006.636106:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766006.636118:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766006/real 1775766006] req@ffffa0e9d0542a00 x1862023861370752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766006.636130:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861370752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766006.636135:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766006.636138:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766006.636140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861370752:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766007.211677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766007.467562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766008.235783:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766008.491612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766009.259639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766009.515627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766010.283666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766010.539608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766011.307780:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766011.563632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766011.755660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766011.755669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766011.755674:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766011.755683:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1740 00000100:00080000:2.0:1775766011.755694:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766011.755700:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766011.755753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861370880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766011.755785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766011.755790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766011.755793:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766011.755798:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1740 00000100:00080000:2.0:1775766011.755803:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766011.755808:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766011.755819:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766011.755858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861371008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766011.755906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766011.755910:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861370880:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766011.755912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766011.755915:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766011.755938:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1740 00000100:00080000:2.0:1775766011.755943:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766011.755948:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766011.755956:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766011.755981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861371136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766011.756061:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766011.756140:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766011/real 1775766011] req@ffffa0e9d0713b80 x1862023861370880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766011.756194:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861370880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766011.756208:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766011.756213:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766011.756218:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861370880:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766011.756258:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766011.756345:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766011.756361:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766011/real 1775766011] req@ffffa0e9d0711c00 x1862023861371008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766011.756373:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861371008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766011.756381:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766011.756385:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766011.756388:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371008:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766011.756394:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766011.756412:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766011.756420:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766011/real 1775766011] req@ffffa0e9d0711f80 x1862023861371136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766011.756430:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861371136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766011.756440:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766011.756445:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766011.756450:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371136:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766012.331646:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766012.587655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766013.355700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766013.611681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766014.379656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766014.635651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766015.403585:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766015.659596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766016.427670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766016.683655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766016.875643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766016.875656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766016.875662:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766016.875676:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1745 00000100:00080000:1.0:1775766016.875710:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766016.875718:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766016.875757:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861371264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766016.875790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766016.875794:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766016.875796:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766016.875800:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1745 00000100:00080000:1.0:1775766016.875803:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766016.875806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766016.875816:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766016.875828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861371392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766016.875840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766016.875844:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766016.875846:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766016.875850:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1745 00000100:00080000:1.0:1775766016.875853:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766016.875858:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766016.875864:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861371264:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766016.875864:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766016.875874:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861371520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766016.875977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766016.876040:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766016/real 1775766016] req@ffffa0e9c8ded180 x1862023861371264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766016.876065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861371264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766016.876118:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766016.876124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766016.876130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861371264:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766016.876142:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766016.876177:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766016.876195:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766016/real 1775766016] req@ffffa0e9c8dec000 x1862023861371392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766016.876208:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861371392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766016.876216:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766016.876219:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766016.876223:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371392:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766016.876246:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766016.876265:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766016.876274:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766016/real 1775766016] req@ffffa0e9c8dee300 x1862023861371520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766016.876300:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861371520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766016.876308:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766016.876311:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766016.876314:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371520:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766017.451669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766017.707580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766018.475577:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766018.731607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766019.499704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766019.755579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766020.523611:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766020.779581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766021.547581:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766021.803587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766021.995562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766021.995570:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766021.995574:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766021.995582:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1750 00000100:00080000:1.0:1775766021.995592:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766021.995597:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766021.995643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861371648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766021.995666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766021.995668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766021.995670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766021.995673:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1750 00000100:00080000:1.0:1775766021.995675:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766021.995677:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766021.995685:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766021.995693:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861371776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766021.995699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766021.995701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766021.995703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766021.995705:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1750 00000100:00080000:1.0:1775766021.995707:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766021.995709:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766021.995727:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766021.995735:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861371904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766021.995749:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861371648:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766021.995805:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766021.995846:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766021/real 1775766021] req@ffffa0e9c8ded500 x1862023861371648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766021.995867:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861371648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766021.995875:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766021.995878:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766021.995881:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861371648:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766021.995886:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766021.995924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766021.995934:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766021/real 1775766021] req@ffffa0e9c8dec380 x1862023861371776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766021.995950:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861371776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766021.995956:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766021.995959:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766021.995961:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371776:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766021.995994:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766021.996031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766021.996045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766021/real 1775766021] req@ffffa0e9c8deed80 x1862023861371904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766021.996057:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861371904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766021.996063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766021.996065:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766021.996067:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861371904:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766022.571772:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766022.827589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766023.595609:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766023.851573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766024.619627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766024.875591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766025.643617:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766025.899576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766026.667845:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766026.923649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766027.115586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766027.115595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766027.115599:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766027.115607:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1755 00000100:00080000:1.0:1775766027.115616:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766027.115622:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766027.115662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861372032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766027.115693:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766027.115697:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766027.115700:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766027.115704:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1755 00000100:00080000:1.0:1775766027.115708:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766027.115713:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766027.115724:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766027.115738:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861372160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766027.115749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766027.115765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766027.115767:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766027.115770:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1755 00000100:00080000:1.0:1775766027.115775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766027.115779:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766027.115785:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766027.115793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861372288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766027.115832:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861372032:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766027.116016:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372160:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766027.116082:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372288:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766027.691581:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766027.947577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766028.715691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766028.971606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766029.739681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766029.995676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766030.187760:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766030.187803:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766030.187822:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766030.187885:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766030.187898:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766030.187992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766027/real 1775766030] req@ffffa0e9c8def800 x1862023861372032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766030.188027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861372032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766030.188053:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766030.188060:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766030.188065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861372032:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766030.188076:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766027/real 1775766030] req@ffffa0e9c8def480 x1862023861372160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766030.188095:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861372160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766030.188116:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766030.188121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766030.188126:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372160:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766030.188134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766027/real 1775766030] req@ffffa0e9c8dedc00 x1862023861372288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766030.188150:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861372288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766030.188189:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766030.188195:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766030.188200:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372288:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766030.763630:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766031.019685:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766031.787644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766032.043600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766032.235619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766032.235632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766032.235648:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766032.235674:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1760 00000100:00080000:3.0:1775766032.235689:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766032.235697:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766032.235755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861372416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766032.235788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766032.235791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766032.235793:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766032.235816:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1760 00000100:00080000:3.0:1775766032.235820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766032.235824:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766032.235848:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766032.235865:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861372544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766032.235879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766032.235884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766032.235888:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766032.235890:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861372416:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766032.235906:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1760 00000100:00080000:3.0:1775766032.235913:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766032.235934:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766032.235941:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766032.235956:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861372672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766032.236022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766032.236085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766032/real 1775766032] req@ffffa0e9d0541500 x1862023861372416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766032.236113:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861372416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766032.236128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766032.236133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766032.236138:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861372416:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766032.236171:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766032.236202:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766032.236218:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766032/real 1775766032] req@ffffa0e9d0541880 x1862023861372544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766032.236234:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861372544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766032.236244:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766032.236265:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766032.236269:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372544:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766032.236277:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766032.236306:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766032.236320:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766032/real 1775766032] req@ffffa0e9d0542680 x1862023861372672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766032.236335:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861372672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766032.236344:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766032.236347:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766032.236351:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372672:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766032.811756:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766033.067601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766033.835580:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766034.091581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766034.859667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766035.115628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766035.883872:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766036.139637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766036.907596:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766037.163647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766037.355628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766037.355638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766037.355642:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766037.355651:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1765 00000100:00080000:3.0:1775766037.355661:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766037.355666:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766037.355705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861372800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766037.355754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766037.355758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766037.355760:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766037.355765:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1765 00000100:00080000:3.0:1775766037.355770:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766037.355775:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766037.355787:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766037.355804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861372928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766037.355819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766037.355823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766037.355825:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766037.355831:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1765 00000100:00080000:3.0:1775766037.355847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766037.355853:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766037.355860:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766037.355889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861373056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766037.355924:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861372800:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766037.356036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766037.356100:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766037/real 1775766037] req@ffffa0e9d0542680 x1862023861372800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766037.356125:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861372800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766037.356139:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766037.356162:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766037.356167:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861372800:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766037.356177:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766037.356205:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766037.356219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766037/real 1775766037] req@ffffa0e9d0541880 x1862023861372928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766037.356232:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861372928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766037.356241:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766037.356245:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766037.356248:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861372928:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766037.356281:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766037.356299:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766037.356309:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766037/real 1775766037] req@ffffa0e9d0541500 x1862023861373056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766037.356322:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861373056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766037.356335:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766037.356341:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766037.356346:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373056:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766037.931647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766038.187585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766038.955744:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766039.211585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766039.979613:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766040.235575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766041.003696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766041.259579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766042.027690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766042.283611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766042.475636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766042.475646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766042.475651:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766042.475660:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1770 00000100:00080000:0.0:1775766042.475670:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766042.475677:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766042.475716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861373184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766042.475768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766042.475773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766042.475775:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766042.475779:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1770 00000100:00080000:0.0:1775766042.475783:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766042.475786:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766042.475808:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766042.475854:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861373312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766042.475866:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861373184:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766042.475905:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766042.475911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766042.475913:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766042.475922:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1770 00000100:00080000:0.0:1775766042.475963:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766042.475973:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:1.0:1775766042.475974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:0.0:1775766042.475986:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766042.476011:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861373440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766042.476028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766042/real 1775766042] req@ffffa0e9d09dce00 x1862023861373184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766042.476048:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861373184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766042.476059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766042.476075:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766042.476079:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861373184:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766042.476104:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766042.476123:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766042.476132:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766042/real 1775766042] req@ffffa0e9d09dc380 x1862023861373312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766042.476140:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861373312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766042.476146:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766042.476148:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766042.476150:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373312:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766042.476154:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766042.476166:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766042.476172:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766042/real 1775766042] req@ffffa0e9d09dc000 x1862023861373440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766042.476190:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861373440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766042.476196:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766042.476197:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766042.476199:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373440:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766043.051675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766043.307621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766044.075702:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766044.331578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766045.099633:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766045.355565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766046.123647:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766046.379554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766047.147757:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766047.403620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766047.595647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766047.595655:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766047.595675:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766047.595683:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1776 00000100:00080000:3.0:1775766047.595692:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766047.595697:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766047.595729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861373568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766047.595755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766047.595758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766047.595759:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766047.595762:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1776 00000100:00080000:3.0:1775766047.595764:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766047.595767:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766047.595787:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766047.595805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861373696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766047.595813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766047.595816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766047.595818:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766047.595820:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1776 00000100:00080000:3.0:1775766047.595823:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766047.595827:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766047.595831:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861373568:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775766047.595832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766047.595843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861373824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766047.595893:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766047.595926:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766047/real 1775766047] req@ffffa0e9d0542a00 x1862023861373568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766047.595949:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861373568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766047.595957:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766047.595959:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766047.595962:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861373568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766047.595968:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766047.595982:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766047.595990:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766047/real 1775766047] req@ffffa0e9d0540000 x1862023861373696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766047.595998:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861373696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766047.596003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766047.596005:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766047.596008:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373696:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766047.596025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766047.596035:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766047.596049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766047/real 1775766047] req@ffffa0e9d0543b80 x1862023861373824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766047.596057:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861373824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766047.596063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766047.596065:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766047.596067:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861373824:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766048.171668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766048.427609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766049.195678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766049.451637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766050.219689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766050.475618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766051.243659:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766051.499593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766052.267644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766052.523607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766052.715643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766052.715654:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766052.715659:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766052.715668:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1781 00000100:00080000:1.0:1775766052.715681:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766052.715687:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766052.715747:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861373952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766052.715780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766052.715784:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766052.715785:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766052.715788:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1781 00000100:00080000:1.0:1775766052.715791:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766052.715793:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766052.715824:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766052.715830:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861373952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766052.715838:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861374080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766052.715847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766052.715849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766052.715851:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766052.715854:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1781 00000100:00080000:1.0:1775766052.715857:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766052.715859:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766052.715863:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766052.715871:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861374208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766052.715902:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766052.715945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766052/real 1775766052] req@ffffa0e9c8deea00 x1862023861373952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.715962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861373952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.715970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.715974:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766052.715977:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861373952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766052.716001:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766052.716026:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766052.716034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766052/real 1775766052] req@ffffa0e9c8dee680 x1862023861374080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.716043:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861374080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.716049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.716051:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766052.716053:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374080:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766052.716056:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766052.716066:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766052.716071:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766052/real 1775766052] req@ffffa0e9c8ded880 x1862023861374208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.716078:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861374208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.716084:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.716085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766052.716087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374208:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766053.291710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766053.547659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766054.315648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766054.571651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766055.339687:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766055.595656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766056.363722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766056.619621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766057.387617:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766057.643585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766057.835655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766057.835669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766057.835698:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766057.835729:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1786 00000100:00080000:3.0:1775766057.835749:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766057.835759:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766057.835824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861374336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766057.835873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766057.835877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766057.835879:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766057.835883:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1786 00000100:00080000:3.0:1775766057.835888:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766057.835892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766057.835906:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766057.835927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861374464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766057.835937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766057.835941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766057.835944:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766057.835970:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861374336:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766057.835978:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1786 00000100:00080000:3.0:1775766057.835984:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766057.835990:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766057.836007:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766057.836032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861374592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766057.836081:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766057.836144:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766057/real 1775766057] req@ffffa0e9d0541c00 x1862023861374336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766057.836165:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861374336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766057.836177:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766057.836182:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766057.836186:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861374336:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766057.836195:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766057.836216:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766057.836228:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766057/real 1775766057] req@ffffa0e9d0540380 x1862023861374464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766057.836250:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861374464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766057.836258:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766057.836262:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766057.836265:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374464:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766057.836289:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766057.836316:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766057.836325:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766057/real 1775766057] req@ffffa0e9d0542300 x1862023861374592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766057.836363:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861374592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766057.836377:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766057.836382:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766057.836386:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374592:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766058.411620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766058.667627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766059.435773:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766059.691569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766060.459646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766060.715576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766061.483613:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766061.739586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766062.507658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766062.763599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766062.955580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766062.955588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766062.955599:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766062.955607:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1791 00000100:00080000:3.0:1775766062.955617:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766062.955622:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766062.955679:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861374720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766062.955701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766062.955704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766062.955705:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766062.955708:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1791 00000100:00080000:3.0:1775766062.955711:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766062.955714:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766062.955721:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766062.955729:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861374848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766062.955751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766062.955754:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861374720:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766062.955755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766062.955758:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766062.955766:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1791 00000100:00080000:3.0:1775766062.955769:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766062.955772:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766062.955776:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766062.955786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861374976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766062.955818:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766062.955853:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766062/real 1775766062] req@ffffa0e9d0543800 x1862023861374720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766062.955866:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861374720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766062.955873:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766062.955876:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766062.955879:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861374720:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766062.955895:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766062.955907:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766062.955915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766062/real 1775766062] req@ffffa0e9d0540700 x1862023861374848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766062.955930:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861374848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766062.955935:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766062.955937:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766062.955939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374848:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766062.955952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766062.955966:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766062.955972:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766062/real 1775766062] req@ffffa0e9d0540e00 x1862023861374976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766062.955980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861374976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766062.955985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766062.955987:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766062.955989:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861374976:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766063.531744:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766063.787658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766064.555679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766064.811668:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766065.579757:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766065.835666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766066.603730:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766066.859700:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766067.627668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766067.883670:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766068.075640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766068.075648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766068.075652:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766068.075660:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1796 00000100:00080000:2.0:1775766068.075669:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766068.075674:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766068.075709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861375104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766068.075732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766068.075748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766068.075750:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766068.075753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1796 00000100:00080000:2.0:1775766068.075757:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766068.075760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766068.075769:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766068.075777:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861375104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766068.075780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861375232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766068.075792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766068.075795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766068.075796:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766068.075799:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1796 00000100:00080000:2.0:1775766068.075801:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766068.075802:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766068.075805:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766068.075813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861375360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766068.075847:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766068.075880:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766068/real 1775766068] req@ffffa0e9d0710700 x1862023861375104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766068.075909:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861375104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766068.075917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766068.075920:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766068.075923:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861375104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766068.075942:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766068.075960:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766068.075968:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766068/real 1775766068] req@ffffa0e9d0712d80 x1862023861375232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766068.075976:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861375232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766068.075982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766068.075984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766068.075986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766068.075990:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766068.075999:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766068.076005:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766068/real 1775766068] req@ffffa0e9d0712680 x1862023861375360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766068.076012:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861375360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766068.076026:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766068.076029:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766068.076031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375360:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766068.651746:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766068.907600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766069.675824:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766069.931681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766070.699663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766070.955637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766071.723683:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766071.979629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766072.747641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766073.003633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766073.195749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766073.195770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766073.195778:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766073.195793:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1801 00000100:00080000:3.0:1775766073.195812:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766073.195823:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766073.195898:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861375488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766073.195948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766073.195953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766073.195956:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766073.195992:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1801 00000100:00080000:3.0:1775766073.196002:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766073.196013:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766073.196034:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766073.196057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861375488:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766073.196074:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861375616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766073.196102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766073.196107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766073.196111:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766073.196118:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1801 00000100:00080000:3.0:1775766073.196124:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766073.196131:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766073.196141:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766073.196160:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861375744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766073.196195:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766073.196271:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766073/real 1775766073] req@ffffa0e9d0540a80 x1862023861375488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766073.196308:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861375488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766073.196327:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766073.196336:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766073.196359:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861375488:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766073.196400:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766073.196442:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766073.196482:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766073/real 1775766073] req@ffffa0e9d0542d80 x1862023861375616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766073.196507:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861375616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766073.196625:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766073.196632:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766073.196638:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375616:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766073.196650:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766073.196688:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766073.196707:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766073/real 1775766073] req@ffffa0e9d0541f80 x1862023861375744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766073.196728:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861375744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766073.196743:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766073.196749:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766073.196754:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861375744:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766073.771625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766074.027628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766074.795689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766075.051581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766075.819680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766076.075614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766076.843769:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766077.099550:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766077.867715:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766078.123621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766078.315685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766078.315699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766078.315705:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766078.315717:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1806 00000100:00080000:3.0:1775766078.315732:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766078.315739:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766078.315822:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861375872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766078.315883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766078.315887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766078.315889:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766078.315894:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1806 00000100:00080000:3.0:1775766078.315898:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766078.315902:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766078.315914:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766078.315928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861376000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766078.315948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766078.315953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766078.315956:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766078.315960:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1806 00000100:00080000:3.0:1775766078.315963:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766078.315967:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766078.315971:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766078.315981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861376128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766078.315997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861375872:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766078.316104:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766078.316208:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766078/real 1775766078] req@ffffa0e9d0541180 x1862023861375872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766078.316230:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861375872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766078.316251:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766078.316255:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766078.316260:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861375872:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766078.316268:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766078.316295:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766078.316307:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766078/real 1775766078] req@ffffa0e9d0543480 x1862023861376000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766078.316319:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861376000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766078.316326:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766078.316329:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766078.316332:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376000:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766078.316359:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766078.316374:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766078.316382:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766078/real 1775766078] req@ffffa0e9d0543100 x1862023861376128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766078.316392:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861376128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766078.316410:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766078.316412:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766078.316415:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376128:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766078.891697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766079.147640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766079.915688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766080.171725:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766080.939678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766081.195570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766081.963695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766082.219575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766082.987571:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766083.243595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766083.435610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766083.435620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766083.435625:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766083.435653:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1811 00000100:00080000:3.0:1775766083.435664:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766083.435670:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766083.435706:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861376256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766083.435736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766083.435741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766083.435743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766083.435748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1811 00000100:00080000:3.0:1775766083.435752:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766083.435757:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766083.435768:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766083.435786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861376384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766083.435797:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861376256:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766083.435799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766083.435803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766083.435806:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766083.435811:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1811 00000100:00080000:3.0:1775766083.435816:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766083.435820:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766083.435826:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766083.435836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861376512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766083.435880:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766083.435927:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766083/real 1775766083] req@ffffa0e9d0543100 x1862023861376256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766083.435945:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861376256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766083.435954:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766083.435958:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766083.435962:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861376256:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766083.435983:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766083.436005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766083.436015:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766083/real 1775766083] req@ffffa0e9d0543480 x1862023861376384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766083.436025:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861376384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766083.436031:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766083.436033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766083.436036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376384:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766083.436048:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766083.436065:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766083.436073:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766083/real 1775766083] req@ffffa0e9d0541180 x1862023861376512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766083.436082:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861376512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766083.436088:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766083.436090:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766083.436093:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376512:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766084.011673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766084.267583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766085.035690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766085.291580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766086.059607:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766086.315592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766087.083754:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766087.339666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766088.107625:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766088.363626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766088.555586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766088.555597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766088.555602:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766088.555611:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1817 00000100:00080000:1.0:1775766088.555621:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766088.555627:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766088.555665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861376640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766088.555691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766088.555694:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766088.555696:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766088.555700:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1817 00000100:00080000:1.0:1775766088.555703:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766088.555706:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766088.555722:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766088.555733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861376768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766088.555739:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861376640:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766088.555767:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766088.555770:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766088.555772:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766088.555777:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1817 00000100:00080000:1.0:1775766088.555793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766088.555797:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775766088.555803:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775766088.555803:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766088.555818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861376896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766088.555847:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766088/real 1775766088] req@ffffa0e9c8deca80 x1862023861376640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766088.555864:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861376640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766088.555872:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766088.555875:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766088.555879:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861376640:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766088.555908:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766088.555927:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766088.555937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766088/real 1775766088] req@ffffa0e9c8dedf80 x1862023861376768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766088.555946:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861376768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766088.555953:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766088.555955:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766088.555958:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376768:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766088.555962:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766088.555974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766088.555980:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766088/real 1775766088] req@ffffa0e9c8dece00 x1862023861376896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766088.555988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861376896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766088.555994:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766088.555996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766088.555998:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861376896:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766089.131628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766089.387655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766090.155692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766090.411603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766091.179612:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766091.435558:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766092.203611:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766092.459634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766093.227685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766093.483623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766093.675670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766093.675682:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766093.675689:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766093.675702:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1822 00000100:00080000:1.0:1775766093.675717:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766093.675726:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766093.675793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861377024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766093.675837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766093.675840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766093.675842:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766093.675871:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1822 00000100:00080000:1.0:1775766093.675876:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766093.675880:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766093.675903:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766093.675920:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861377024:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766093.675921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861377152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766093.675940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766093.675944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766093.675945:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766093.675950:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1822 00000100:00080000:1.0:1775766093.675953:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766093.675957:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766093.675961:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766093.675999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861377280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766093.676106:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377152:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766093.676148:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377280:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766094.251733:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766094.507613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766095.275718:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766095.531626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766096.299767:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766096.555603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775766096.747887:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775766096.747935:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766096.747954:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766096.747997:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766096.748007:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766096.748078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766093/real 1775766096] req@ffffa0e9c8defb80 x1862023861377152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766096.748144:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861377152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766096.748164:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766096.748174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766096.748182:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377152:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766096.748197:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766093/real 1775766096] req@ffffa0e9c8def100 x1862023861377280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766096.748223:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861377280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766096.748253:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766096.748258:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766096.748264:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377280:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766096.748274:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766093/real 1775766096] req@ffffa0e9c8dec700 x1862023861377024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766096.748350:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861377024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766096.748364:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766096.748370:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766096.748375:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861377024:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775766097.323777:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766097.579657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766098.347620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766098.603704:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766098.795727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766098.795745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766098.795753:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766098.795768:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1827 00000100:00080000:3.0:1775766098.795782:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766098.795790:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766098.795835:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861377408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766098.795881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766098.795886:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766098.795889:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766098.795893:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1827 00000100:00080000:3.0:1775766098.795898:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766098.795914:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766098.795951:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766098.795970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861377536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766098.795988:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766098.795997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861377408:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766098.795998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766098.796003:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766098.796027:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1827 00000100:00080000:3.0:1775766098.796033:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766098.796038:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766098.796045:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766098.796062:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861377664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766098.796138:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766098.796206:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766098/real 1775766098] req@ffffa0e9d0541f80 x1862023861377408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766098.796241:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861377408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766098.796261:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766098.796270:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766098.796278:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861377408:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766098.796352:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766098.796427:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766098.796455:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766098/real 1775766098] req@ffffa0e9d0542d80 x1862023861377536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766098.796481:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861377536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766098.796564:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766098.796572:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766098.796578:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377536:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766098.796591:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766098.796635:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766098.796659:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766098/real 1775766098] req@ffffa0e9d0540a80 x1862023861377664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766098.796683:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861377664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766098.796699:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766098.796718:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766098.796726:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377664:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766099.371686:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766099.627653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766100.395822:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766100.651642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766101.419615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766101.675572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766102.443676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766102.699634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766103.467721:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766103.723641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766103.915606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766103.915624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766103.915631:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766103.915647:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1832 00000100:00080000:3.0:1775766103.915663:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766103.915676:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766103.915732:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861377792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766103.915771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766103.915776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766103.915799:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766103.915805:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1832 00000100:00080000:3.0:1775766103.915811:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766103.915816:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766103.915829:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766103.915845:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861377920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766103.915860:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766103.915865:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766103.915868:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766103.915874:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1832 00000100:00080000:3.0:1775766103.915878:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766103.915883:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766103.915889:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766103.915906:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861378048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766103.915908:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861377792:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766103.916002:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766103.916077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766103/real 1775766103] req@ffffa0e9d0540e00 x1862023861377792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766103.916117:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861377792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766103.916154:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766103.916164:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766103.916174:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861377792:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766103.916188:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766103.916224:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766103.916303:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766103/real 1775766103] req@ffffa0e9d0540700 x1862023861377920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766103.916321:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861377920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766103.916331:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766103.916335:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766103.916339:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861377920:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766103.916369:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766103.916395:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766103.916406:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766103/real 1775766103] req@ffffa0e9d0543800 x1862023861378048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766103.916420:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861378048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766103.916430:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766103.916433:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766103.916436:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378048:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766104.491635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766104.747598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766105.515747:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766105.771566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766106.539684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766106.795738:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766107.563632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766107.819574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766108.587722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766108.843585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766109.035613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766109.035665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766109.035670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766109.035682:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1837 00000100:00080000:1.0:1775766109.035713:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766109.035720:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766109.035762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861378176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766109.035788:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766109.035791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766109.035793:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766109.035796:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1837 00000100:00080000:1.0:1775766109.035798:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766109.035800:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766109.035810:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766109.035818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861378304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766109.035826:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766109.035829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766109.035830:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766109.035833:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1837 00000100:00080000:1.0:1775766109.035835:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766109.035837:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766109.035840:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766109.035848:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861378176:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766109.035848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861378432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766109.035922:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766109.035984:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766109/real 1775766109] req@ffffa0e9c8dec700 x1862023861378176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766109.036005:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861378176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766109.036016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766109.036020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766109.036024:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861378176:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766109.036031:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766109.036052:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766109.036063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766109/real 1775766109] req@ffffa0e9c8def100 x1862023861378304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766109.036073:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861378304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766109.036080:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766109.036082:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766109.036085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378304:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766109.036122:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766109.036141:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766109.036150:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766109/real 1775766109] req@ffffa0e9c8defb80 x1862023861378432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766109.036161:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861378432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766109.036168:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766109.036170:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766109.036173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378432:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766109.611640:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766109.867582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766110.635614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766110.891613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766111.659724:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766111.915560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766112.683735:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766112.939597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766113.707618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766113.963619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766114.155617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766114.155628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766114.155634:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766114.155644:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1842 00000100:00080000:3.0:1775766114.155657:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766114.155686:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766114.155725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861378560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766114.155753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766114.155756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766114.155757:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766114.155760:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1842 00000100:00080000:3.0:1775766114.155763:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766114.155766:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766114.155776:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766114.155785:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861378688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766114.155793:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766114.155796:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861378560:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766114.155796:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766114.155798:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766114.155805:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1842 00000100:00080000:3.0:1775766114.155807:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766114.155810:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766114.155812:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766114.155819:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861378816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766114.155871:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766114.155923:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766114/real 1775766114] req@ffffa0e9d0542300 x1862023861378560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766114.155941:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861378560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766114.155951:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766114.155954:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766114.155958:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861378560:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766114.155993:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766114.156019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766114.156033:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766114/real 1775766114] req@ffffa0e9d0540380 x1862023861378688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766114.156047:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861378688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766114.156055:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766114.156058:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766114.156061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378688:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766114.156067:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766114.156085:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766114.156096:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766114/real 1775766114] req@ffffa0e9d0541c00 x1862023861378816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766114.156108:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861378816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766114.156117:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766114.156120:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766114.156124:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861378816:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766114.731659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766114.987599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766115.755667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766116.011705:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766116.779716:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766117.035639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766117.803707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766118.059583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766118.827656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766119.083646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766119.275624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766119.275673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766119.275680:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766119.275696:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1847 00000100:00080000:3.0:1775766119.275713:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766119.275724:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766119.275780:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861378944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766119.275835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766119.275840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766119.275842:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766119.275847:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1847 00000100:00080000:3.0:1775766119.275851:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766119.275855:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766119.275870:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766119.275887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861379072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766119.275900:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766119.275905:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766119.275917:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766119.275921:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1847 00000100:00080000:3.0:1775766119.275925:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766119.275929:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766119.275934:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766119.275946:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861378944:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766119.275948:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861379200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766119.276066:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766119.276176:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766119/real 1775766119] req@ffffa0e9d0543b80 x1862023861378944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766119.276209:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861378944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766119.276225:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766119.276230:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766119.276236:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861378944:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766119.276246:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766119.276278:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766119.276293:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766119/real 1775766119] req@ffffa0e9d0540000 x1862023861379072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766119.276307:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861379072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766119.276317:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766119.276321:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766119.276325:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379072:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766119.276356:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766119.276375:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766119.276385:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766119/real 1775766119] req@ffffa0e9d0542a00 x1862023861379200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766119.276413:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861379200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766119.276423:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766119.276426:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766119.276430:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379200:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766119.851726:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766120.107660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766120.875638:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766121.131621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766121.899724:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766122.155672:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766122.923693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766123.179677:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766123.947730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766124.203648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766124.395615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766124.395626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766124.395632:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766124.395643:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1852 00000100:00080000:3.0:1775766124.395656:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766124.395663:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766124.395710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861379328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766124.395738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766124.395741:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766124.395743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766124.395745:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1852 00000100:00080000:3.0:1775766124.395749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766124.395752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766124.395763:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766124.395775:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861379328:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766124.395775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861379456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766124.395792:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766124.395795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766124.395796:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766124.395799:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1852 00000100:00080000:3.0:1775766124.395820:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766124.395824:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766124.395829:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766124.395838:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861379584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766124.395861:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766124.395915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766124/real 1775766124] req@ffffa0e9d0541500 x1862023861379328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766124.395940:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861379328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766124.395954:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766124.395960:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766124.395966:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861379328:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766124.395996:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766124.396018:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766124.396028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766124/real 1775766124] req@ffffa0e9d0541880 x1862023861379456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766124.396065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861379456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766124.396072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766124.396075:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766124.396078:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379456:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766124.396083:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766124.396100:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766124.396108:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766124/real 1775766124] req@ffffa0e9d0542680 x1862023861379584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766124.396116:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861379584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766124.396122:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766124.396123:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766124.396125:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379584:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766124.971707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766125.227557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766125.995671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766126.251720:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766127.019660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766127.275647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766128.043679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766128.299661:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766129.067643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766129.323651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766129.515585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766129.515596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766129.515602:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766129.515613:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1858 00000100:00080000:1.0:1775766129.515625:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766129.515633:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766129.515680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861379712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766129.515719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766129.515722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766129.515723:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766129.515726:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1858 00000100:00080000:1.0:1775766129.515729:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766129.515731:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766129.515741:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766129.515773:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861379840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766129.515774:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861379712:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766129.515787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766129.515789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766129.515790:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766129.515797:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1858 00000100:00080000:1.0:1775766129.515802:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766129.515805:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766129.515811:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766129.515823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861379968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766129.515876:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766129.515935:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766129/real 1775766129] req@ffffa0e9c8dece00 x1862023861379712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766129.515957:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861379712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766129.515978:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766129.515981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766129.515986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861379712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766129.516029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766129.516065:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766129.516076:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766129/real 1775766129] req@ffffa0e9c8dedf80 x1862023861379840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766129.516085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861379840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766129.516091:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766129.516093:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766129.516095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379840:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766129.516100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766129.516111:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766129.516117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766129/real 1775766129] req@ffffa0e9c8deca80 x1862023861379968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766129.516135:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861379968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766129.516141:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766129.516143:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766129.516145:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861379968:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766130.091690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766130.347646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766131.115622:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766131.371575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766132.139609:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766132.395624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766133.163676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766133.419596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766134.187711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766134.443579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766134.635595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766134.635606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766134.635611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766134.635622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1863 00000100:00080000:1.0:1775766134.635635:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766134.635641:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766134.635686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861380096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766134.635739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766134.635743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766134.635746:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766134.635775:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1863 00000100:00080000:1.0:1775766134.635781:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766134.635786:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766134.635801:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766134.635810:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861380096:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766134.635821:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861380224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766134.635835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766134.635838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766134.635840:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766134.635843:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1863 00000100:00080000:1.0:1775766134.635847:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766134.635850:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766134.635855:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766134.635866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861380352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766134.635927:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766134.636014:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766134/real 1775766134] req@ffffa0e9c8ded880 x1862023861380096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766134.636031:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861380096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766134.636039:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766134.636043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766134.636053:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861380096:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766134.636077:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380224:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766134.636095:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766134.636103:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766134/real 1775766134] req@ffffa0e9c8dee680 x1862023861380224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766134.636112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861380224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766134.636129:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766134.636133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766134.636136:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766134.636143:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766134.636167:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766134.636179:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766134/real 1775766134] req@ffffa0e9c8deea00 x1862023861380352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766134.636187:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861380352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766134.636193:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766134.636195:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766134.636197:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380352:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766135.211625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766135.467611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766136.235691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766136.491579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766137.259671:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766137.515625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766138.283615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766138.539566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766139.307728:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766139.563635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766139.755689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766139.755700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766139.755705:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766139.755716:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1868 00000100:00080000:0.0:1775766139.755729:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766139.755735:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766139.755787:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861380480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766139.755823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766139.755848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766139.755851:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766139.755855:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861380480:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766139.755857:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1868 00000100:00080000:0.0:1775766139.755865:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766139.755872:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766139.755885:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766139.755919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861380608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766139.755933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766139.755937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766139.755939:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766139.755942:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1868 00000100:00080000:0.0:1775766139.755947:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766139.755948:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775766139.755952:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766139.755957:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766139.755967:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861380736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766139.756016:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766139/real 1775766139] req@ffffa0e9d09df480 x1862023861380480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766139.756041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861380480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766139.756067:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766139.756072:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766139.756075:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861380480:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766139.756100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766139.756120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766139.756130:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766139/real 1775766139] req@ffffa0e9d09de300 x1862023861380608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766139.756139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861380608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766139.756145:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766139.756147:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766139.756149:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380608:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766139.756153:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766139.756164:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766139.756170:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766139/real 1775766139] req@ffffa0e9d09dea00 x1862023861380736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766139.756177:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861380736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766139.756182:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766139.756184:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766139.756186:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380736:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766140.331627:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766140.587612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766141.355645:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766141.611586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766142.379621:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766142.635567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766143.403637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766143.659598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766144.427614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766144.683609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766144.875661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766144.875674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766144.875682:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766144.875697:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1873 00000100:00080000:0.0:1775766144.875717:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766144.875765:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766144.875833:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861380864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766144.875900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766144.875904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766144.875907:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766144.875912:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1873 00000100:00080000:0.0:1775766144.875919:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766144.875926:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766144.875945:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766144.875961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861380864:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766144.875976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861380992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766144.875999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766144.876006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766144.876010:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766144.876017:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1873 00000100:00080000:0.0:1775766144.876060:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766144.876069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775766144.876076:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766144.876085:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766144.876108:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861381120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766144.876161:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766144/real 1775766144] req@ffffa0e9d09dea00 x1862023861380864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766144.876200:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861380864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766144.876217:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766144.876223:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766144.876228:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861380864:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766144.876264:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766144.876310:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766144.876336:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766144/real 1775766144] req@ffffa0e9d09de300 x1862023861380992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766144.876370:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861380992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766144.876387:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766144.876394:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766144.876399:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861380992:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766144.876415:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766144.876455:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766144.876550:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766144/real 1775766144] req@ffffa0e9d09df480 x1862023861381120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766144.876575:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861381120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766144.876598:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766144.876603:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766144.876607:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381120:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766145.451661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766145.707599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766146.475688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766146.731563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766147.499683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766147.755580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766148.523705:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766148.779590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766149.547698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766149.803671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766149.995667:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766149.995679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766149.995685:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766149.995696:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1878 00000100:00080000:1.0:1775766149.995709:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766149.995716:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766149.995775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861381248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766149.995834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766149.995837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766149.995839:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766149.995842:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1878 00000100:00080000:1.0:1775766149.995845:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766149.995848:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766149.995859:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766149.995891:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861381376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766149.995899:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861381248:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766149.995903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766149.995907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766149.995909:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766149.995912:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1878 00000100:00080000:1.0:1775766149.995915:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766149.995918:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766149.995921:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766149.995931:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861381504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766149.995982:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766149.996035:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766149/real 1775766149] req@ffffa0e9c8dedc00 x1862023861381248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766149.996054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861381248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766149.996063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766149.996066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766149.996070:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861381248:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766149.996103:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766149.996118:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766149.996126:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766149/real 1775766149] req@ffffa0e9c8def480 x1862023861381376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766149.996134:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861381376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766149.996139:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766149.996141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766149.996144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381376:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766149.996147:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766149.996157:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766149.996162:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766149/real 1775766149] req@ffffa0e9c8def800 x1862023861381504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766149.996181:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861381504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766149.996187:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766149.996189:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766149.996191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381504:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766150.571652:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766150.827592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766151.595682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766151.851612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766152.619797:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766152.875620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766153.643641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766153.899613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766154.667619:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766154.923563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766155.115595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766155.115604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766155.115608:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766155.115616:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1883 00000100:00080000:0.0:1775766155.115625:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766155.115631:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766155.115675:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861381632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766155.115707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766155.115710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766155.115712:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766155.115716:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1883 00000100:00080000:0.0:1775766155.115719:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766155.115722:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766155.115731:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766155.115735:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861381632:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766155.115742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861381760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766155.115778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766155.115781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766155.115783:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766155.115786:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1883 00000100:00080000:0.0:1775766155.115795:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766155.115798:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766155.115804:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766155.115813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861381888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766155.115826:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766155.115863:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766155/real 1775766155] req@ffffa0e9d09dc000 x1862023861381632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766155.115890:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861381632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766155.115898:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766155.115902:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766155.115905:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861381632:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766155.115928:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766155.115961:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766155.115973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766155/real 1775766155] req@ffffa0e9d09dc380 x1862023861381760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766155.115988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861381760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766155.116007:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766155.116012:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766155.116016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381760:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766155.116023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766155.116046:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766155.116055:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766155/real 1775766155] req@ffffa0e9d09dce00 x1862023861381888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766155.116064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861381888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766155.116070:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766155.116073:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766155.116075:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861381888:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766155.691727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766155.947632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766156.715637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766156.971568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766157.739670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766157.995616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766158.763764:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766159.019636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766159.787644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766160.043587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766160.235649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766160.235663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766160.235685:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766160.235753:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1888 00000100:00080000:0.0:1775766160.235769:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766160.235776:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766160.235832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861382016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766160.235880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766160.235885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766160.235887:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766160.235893:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1888 00000100:00080000:0.0:1775766160.235921:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766160.235926:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861382016:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766160.235930:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766160.235947:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766160.235966:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861382144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766160.235978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766160.235982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766160.235984:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766160.235987:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1888 00000100:00080000:0.0:1775766160.236002:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766160.236006:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766160.236009:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766160.236019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861382272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766160.236065:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382144:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766160.236100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382272:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766160.811766:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766161.067625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766161.835755:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766162.091611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766162.859724:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766163.115692:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766163.308058:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766163.308143:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766163.308193:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766163.308264:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766163.308282:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775766163.308600:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766160/real 1775766163] req@ffffa0e9d09df100 x1862023861382144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766163.308654:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861382144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766163.308695:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766163.308705:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766163.308713:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382144:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766163.308729:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766160/real 1775766163] req@ffffa0e9d09dfb80 x1862023861382272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766163.308753:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861382272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766163.308766:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766163.308772:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766163.308777:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766163.308789:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766160/real 1775766163] req@ffffa0e9d09ded80 x1862023861382016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766163.308811:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861382016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766163.308825:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766163.308832:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766163.308873:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861382016:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775766163.883791:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766164.139621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766164.907750:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766165.163669:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766165.355719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766165.355748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766165.355752:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766165.355764:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1893 00000100:00080000:2.0:1775766165.355777:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766165.355783:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766165.355824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861382400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766165.355853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766165.355856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766165.355858:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766165.355860:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1893 00000100:00080000:2.0:1775766165.355863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766165.355866:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766165.355877:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766165.355887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861382528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766165.355895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766165.355898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766165.355900:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766165.355904:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1893 00000100:00080000:2.0:1775766165.355908:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766165.355912:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766165.355917:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766165.355932:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861382656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766165.355982:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861382400:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766165.356088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766165.356207:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766165/real 1775766165] req@ffffa0e9d0710a80 x1862023861382400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766165.356255:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861382400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766165.356276:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766165.356284:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766165.356311:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861382400:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766165.356326:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766165.356372:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766165.356398:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766165/real 1775766165] req@ffffa0e9d0710e00 x1862023861382528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766165.356417:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861382528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766165.356430:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766165.356436:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766165.356441:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382528:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766165.356453:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766165.356612:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766165.356640:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766165/real 1775766165] req@ffffa0e9d0712a00 x1862023861382656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766165.356702:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861382656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766165.356715:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766165.356720:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766165.356725:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382656:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766165.931638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766166.187705:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766166.955776:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766167.211664:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766167.979777:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766168.235697:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766169.003593:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766169.259642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766170.027690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766170.283646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766170.475586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766170.475594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766170.475598:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766170.475619:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1898 00000100:00080000:2.0:1775766170.475628:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766170.475634:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766170.475674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861382784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766170.475705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766170.475709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766170.475712:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766170.475715:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1898 00000100:00080000:2.0:1775766170.475718:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766170.475721:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766170.475729:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766170.475741:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861382912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766170.475752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766170.475756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775766170.475772:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861382784:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766170.475772:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766170.475788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1898 00000100:00080000:2.0:1775766170.475793:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766170.475796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766170.475801:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766170.475812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861383040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766170.475883:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766170.475958:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766170/real 1775766170] req@ffffa0e9d0713480 x1862023861382784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766170.475981:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861382784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766170.475993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766170.475997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766170.476001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861382784:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766170.476030:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766170.476058:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766170.476075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766170/real 1775766170] req@ffffa0e9d0711500 x1862023861382912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766170.476111:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861382912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766170.476124:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766170.476130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766170.476135:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861382912:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766170.476142:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766170.476182:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766170.476194:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766170/real 1775766170] req@ffffa0e9d0712300 x1862023861383040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766170.476205:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861383040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766170.476212:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766170.476216:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766170.476218:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383040:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766171.051705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766171.307643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766172.075731:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766172.331630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766173.099728:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766173.355630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766174.123711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766174.379560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766175.147607:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766175.403669:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766175.595701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766175.595711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766175.595716:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766175.595727:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1904 00000100:00080000:0.0:1775766175.595739:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766175.595746:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766175.595812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861383168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766175.595838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766175.595841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766175.595842:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766175.595844:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1904 00000100:00080000:0.0:1775766175.595847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766175.595851:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766175.595861:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766175.595870:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861383296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766175.595878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766175.595881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766175.595884:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766175.595887:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1904 00000100:00100000:1.0:1775766175.595887:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861383168:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766175.595893:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766175.595896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766175.595899:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766175.595907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861383424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766175.595962:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766175.596004:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766175/real 1775766175] req@ffffa0e9d09dd880 x1862023861383168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766175.596021:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861383168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766175.596036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766175.596040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766175.596043:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861383168:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766175.596066:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766175.596083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766175.596092:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766175/real 1775766175] req@ffffa0e9d09dc700 x1862023861383296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766175.596100:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861383296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766175.596106:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766175.596108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766175.596110:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383296:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766175.596114:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766175.596124:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766175.596130:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766175/real 1775766175] req@ffffa0e9d09df800 x1862023861383424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766175.596137:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861383424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766175.596142:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766175.596144:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766175.596145:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383424:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766176.171732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766176.427599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766177.195737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766177.451622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766178.219689:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766178.475631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766179.243635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766179.499587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766180.267740:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766180.523656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766180.715657:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766180.715694:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766180.715699:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766180.715710:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1909 00000100:00080000:0.0:1775766180.715723:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766180.715731:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766180.715788:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861383552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766180.715819:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766180.715822:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766180.715824:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766180.715826:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1909 00000100:00080000:0.0:1775766180.715829:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766180.715831:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766180.715842:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766180.715850:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861383680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766180.715857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766180.715860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766180.715861:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766180.715863:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1909 00000100:00080000:0.0:1775766180.715865:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766180.715867:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766180.715870:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766180.715878:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861383808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766180.715903:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861383552:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766180.716004:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766180.716068:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766180/real 1775766180] req@ffffa0e9d09dd500 x1862023861383552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766180.716086:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861383552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766180.716095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766180.716098:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766180.716101:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861383552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766180.716107:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766180.716125:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766180.716134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766180/real 1775766180] req@ffffa0e9d09dd180 x1862023861383680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766180.716142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861383680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766180.716147:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766180.716149:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766180.716151:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383680:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766180.716172:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766180.716194:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766180.716201:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766180/real 1775766180] req@ffffa0e9d09ddc00 x1862023861383808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766180.716209:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861383808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766180.716214:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766180.716227:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766180.716229:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861383808:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766181.291646:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766181.547611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766182.315674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766182.571620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766183.339730:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766183.595533:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766184.363678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766184.619682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766185.387714:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766185.643642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766185.835587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766185.835599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766185.835605:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766185.835617:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1914 00000100:00080000:0.0:1775766185.835638:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766185.835646:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766185.835696:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861383936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766185.835728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766185.835732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766185.835733:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766185.835737:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1914 00000100:00080000:0.0:1775766185.835740:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766185.835743:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766185.835753:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766185.835764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861384064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766185.835803:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766185.835807:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766185.835809:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766185.835816:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1914 00000100:00100000:1.0:1775766185.835818:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861383936:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766185.835819:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766185.835846:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766185.835852:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766185.835866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861384192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766185.835913:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766185.835978:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766185/real 1775766185] req@ffffa0e9d09ddf80 x1862023861383936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766185.836017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861383936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766185.836028:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766185.836032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766185.836036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861383936:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766185.836063:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766185.836083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766185.836093:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766185/real 1775766185] req@ffffa0e9d09dca80 x1862023861384064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766185.836113:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861384064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766185.836121:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766185.836123:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766185.836126:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384064:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766185.836130:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766185.836146:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766185.836153:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766185/real 1775766185] req@ffffa0e9d09de680 x1862023861384192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766185.836163:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861384192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766185.836169:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766185.836171:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766185.836173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384192:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766186.411651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766186.667681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766187.435712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766187.691615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766188.459719:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766188.715640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766189.483630:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766189.739619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766190.507634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766190.763612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766190.955576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766190.955584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766190.955587:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766190.955595:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1919 00000100:00080000:0.0:1775766190.955603:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766190.955608:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766190.955642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861384320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766190.955678:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766190.955681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766190.955682:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766190.955685:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1919 00000100:00080000:0.0:1775766190.955687:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766190.955690:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766190.955697:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766190.955711:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861384448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766190.955718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766190.955733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766190.955735:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766190.955738:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1919 00000100:00080000:0.0:1775766190.955741:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766190.955744:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766190.955748:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766190.955750:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861384320:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766190.955758:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861384576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766190.955809:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766190.955854:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766190/real 1775766190] req@ffffa0e9d09de680 x1862023861384320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766190.955876:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861384320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766190.955896:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766190.955900:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766190.955903:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861384320:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766190.955909:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766190.955929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766190.955939:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766190/real 1775766190] req@ffffa0e9d09dca80 x1862023861384448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766190.955947:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861384448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766190.955953:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766190.955955:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766190.955957:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384448:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766190.955974:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766190.955986:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766190.955992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766190/real 1775766190] req@ffffa0e9d09ddf80 x1862023861384576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766190.955999:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861384576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766190.956004:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766190.956006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766190.956008:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384576:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766191.531609:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766191.787656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766192.555642:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766192.811569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766193.579625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766193.835564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766194.603597:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766194.859582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766195.627741:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766195.883562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766196.075615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766196.075626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766196.075631:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766196.075641:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1924 00000100:00080000:0.0:1775766196.075652:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766196.075658:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766196.075700:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861384704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766196.075734:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766196.075737:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766196.075739:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766196.075743:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1924 00000100:00080000:0.0:1775766196.075746:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766196.075749:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766196.075776:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766196.075789:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861384832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766196.075801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766196.075804:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861384704:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766196.075806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766196.075810:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766196.075818:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1924 00000100:00080000:0.0:1775766196.075822:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766196.075826:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766196.075830:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766196.075841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861384960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766196.075900:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766196.075971:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766196/real 1775766196] req@ffffa0e9d09ddc00 x1862023861384704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766196.075994:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861384704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766196.076005:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766196.076009:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766196.076013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861384704:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766196.076035:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766196.076056:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766196.076075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766196/real 1775766196] req@ffffa0e9d09dd180 x1862023861384832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766196.076086:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861384832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766196.076094:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766196.076097:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766196.076100:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384832:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766196.076105:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766196.076137:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766196.076147:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766196/real 1775766196] req@ffffa0e9d09dd500 x1862023861384960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766196.076169:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861384960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766196.076176:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766196.076179:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766196.076181:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861384960:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766196.651601:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766196.907601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766197.675697:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766197.931608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766198.699629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766198.955601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766199.723717:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766199.979613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766200.747628:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766201.003555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766201.195608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766201.195619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766201.195625:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766201.195635:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1929 00000100:00080000:0.0:1775766201.195647:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766201.195654:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766201.195697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861385088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766201.195723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766201.195726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766201.195742:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766201.195745:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1929 00000100:00080000:0.0:1775766201.195748:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766201.195752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766201.195760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766201.195770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861385216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766201.195779:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766201.195782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766201.195784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766201.195787:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1929 00000100:00080000:0.0:1775766201.195790:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766201.195795:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766201.195799:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766201.195808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861385344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766201.195847:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861385088:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766201.195960:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766201.196041:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766201/real 1775766201] req@ffffa0e9d09df800 x1862023861385088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766201.196070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861385088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766201.196085:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766201.196091:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766201.196096:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861385088:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766201.196107:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766201.196136:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766201.196151:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766201/real 1775766201] req@ffffa0e9d09dc700 x1862023861385216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766201.196188:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861385216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766201.196199:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766201.196203:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766201.196207:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385216:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766201.196243:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766201.196270:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766201.196282:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766201/real 1775766201] req@ffffa0e9d09dd880 x1862023861385344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766201.196297:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861385344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766201.196369:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766201.196374:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766201.196378:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385344:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766201.771713:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766202.027591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766202.795740:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766203.051556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766203.819669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766204.075618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766204.843719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766205.099642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766205.867661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766206.123725:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766206.315644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766206.315657:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766206.315664:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766206.315676:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1934 00000100:00080000:1.0:1775766206.315690:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766206.315698:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766206.315768:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861385472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766206.315805:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766206.315808:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766206.315810:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766206.315814:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1934 00000100:00080000:1.0:1775766206.315818:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766206.315822:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766206.315835:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766206.315846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861385600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766206.315859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861385472:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766206.315862:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766206.315878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766206.315880:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766206.315885:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1934 00000100:00080000:1.0:1775766206.315887:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766206.315891:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766206.315895:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766206.315905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861385728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766206.315951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766206.316011:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766206/real 1775766206] req@ffffa0e9c8deed80 x1862023861385472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766206.316058:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861385472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766206.316089:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766206.316094:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766206.316099:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861385472:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766206.316126:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766206.316155:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766206.316185:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766206/real 1775766206] req@ffffa0e9c8dec380 x1862023861385600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766206.316196:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861385600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766206.316204:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766206.316207:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766206.316210:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385600:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766206.316215:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766206.316230:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766206.316237:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766206/real 1775766206] req@ffffa0e9c8ded500 x1862023861385728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766206.316247:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861385728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766206.316253:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766206.316256:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766206.316276:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385728:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766206.891732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766207.147588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766207.915692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766208.171618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766208.939870:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766209.195639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766209.963769:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766210.219611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766210.987636:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766211.243652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766211.435632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766211.435674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766211.435679:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766211.435702:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1939 00000100:00080000:1.0:1775766211.435717:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766211.435725:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766211.435768:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861385856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766211.435799:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766211.435802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766211.435804:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766211.435807:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1939 00000100:00080000:1.0:1775766211.435809:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766211.435812:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766211.435824:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766211.435832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861385984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766211.435839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766211.435842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766211.435844:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766211.435846:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1939 00000100:00080000:1.0:1775766211.435849:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766211.435853:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766211.435856:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766211.435874:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861386112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766211.435891:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861385856:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766211.435971:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766211.436025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766211/real 1775766211] req@ffffa0e9c8dee300 x1862023861385856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766211.436044:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861385856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766211.436054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766211.436058:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766211.436061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861385856:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766211.436067:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766211.436087:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766211.436099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766211/real 1775766211] req@ffffa0e9c8dec000 x1862023861385984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766211.436112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861385984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766211.436121:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766211.436125:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766211.436129:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861385984:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766211.436168:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766211.436198:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766211.436207:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766211/real 1775766211] req@ffffa0e9c8ded180 x1862023861386112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766211.436234:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861386112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766211.436240:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766211.436253:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766211.436255:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386112:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766212.011608:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766212.267577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766213.035830:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766213.291560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766214.059754:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766214.315595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766215.083607:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766215.339587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766216.107675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766216.363566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766216.555615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766216.555626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766216.555632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766216.555642:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1945 00000100:00080000:0.0:1775766216.555654:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766216.555661:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766216.555705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861386240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766216.555754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766216.555759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766216.555762:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766216.555767:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1945 00000100:00080000:0.0:1775766216.555773:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766216.555780:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766216.555822:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766216.555827:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861386240:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766216.555844:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861386368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766216.555862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766216.555867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766216.555869:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766216.555874:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1945 00000100:00080000:0.0:1775766216.555881:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766216.555887:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766216.555893:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766216.555907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861386496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766216.555937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766216.556000:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766216/real 1775766216] req@ffffa0e9d09ded80 x1862023861386240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766216.556027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861386240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766216.556041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766216.556046:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766216.556052:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861386240:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766216.556097:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766216.556126:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766216.556145:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766216/real 1775766216] req@ffffa0e9d09dfb80 x1862023861386368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766216.556202:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861386368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766216.556209:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766216.556211:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766216.556214:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386368:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766216.556219:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766216.556242:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766216.556250:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766216/real 1775766216] req@ffffa0e9d09df100 x1862023861386496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766216.556259:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861386496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766216.556264:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766216.556267:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766216.556269:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386496:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766217.131695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766217.387600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766218.155686:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766218.411603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766219.179715:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766219.435629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766220.203672:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766220.459612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766221.227649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766221.483654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766221.675654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766221.675666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766221.675673:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766221.675686:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1950 00000100:00080000:1.0:1775766221.675702:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766221.675711:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766221.675768:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861386624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766221.675807:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766221.675811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766221.675813:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766221.675818:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1950 00000100:00080000:1.0:1775766221.675821:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766221.675825:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766221.675862:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766221.675865:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861386624:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766221.675880:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861386752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766221.675893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766221.675897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766221.675899:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766221.675903:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1950 00000100:00080000:1.0:1775766221.675907:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766221.675911:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766221.675916:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766221.675927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861386880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766221.675963:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766221.676027:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766221/real 1775766221] req@ffffa0e9c8ded180 x1862023861386624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766221.676052:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861386624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766221.676065:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766221.676069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766221.676074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861386624:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766221.676131:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766221.676161:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766221.676175:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766221/real 1775766221] req@ffffa0e9c8dec000 x1862023861386752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766221.676188:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861386752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766221.676197:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766221.676200:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766221.676204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386752:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766221.676210:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766221.676225:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766221.676233:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766221/real 1775766221] req@ffffa0e9c8dee300 x1862023861386880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766221.676243:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861386880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766221.676251:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766221.676253:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766221.676256:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861386880:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766222.251691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766222.507615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766223.275775:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766223.531625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766224.299632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766224.555571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766225.323742:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766225.579673:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766226.347729:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766226.603621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766226.795619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766226.795644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766226.795648:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766226.795656:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1955 00000100:00080000:2.0:1775766226.795666:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766226.795671:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766226.795702:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861387008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766226.795727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766226.795730:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766226.795731:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766226.795734:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1955 00000100:00080000:2.0:1775766226.795736:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766226.795739:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766226.795746:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766226.795753:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861387136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766226.795772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766226.795775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766226.795776:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766226.795778:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1955 00000100:00080000:2.0:1775766226.795780:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766226.795783:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766226.795789:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766226.795797:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861387264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766226.795844:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861387008:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766226.796016:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387136:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766226.796053:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387264:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766227.371596:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766227.627586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766228.395664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766228.651575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766229.419656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766229.675658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775766229.867803:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775766229.867861:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766229.867882:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766229.867931:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766229.867944:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766229.867984:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766226/real 1775766229] req@ffffa0e9d0712300 x1862023861387264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.868020:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861387264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.868050:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.868057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766229.868061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766229.868071:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766226/real 1775766229] req@ffffa0e9d0713100 x1862023861387008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.868087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861387008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.868096:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.868100:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766229.868104:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861387008:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766229.868110:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766226/real 1775766229] req@ffffa0e9d0711880 x1862023861387136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.868120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861387136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.868156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.868160:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766229.868164:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387136:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766230.443602:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766230.699621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766231.467713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766231.723581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766231.915587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766231.915596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766231.915600:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766231.915608:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1960 00000100:00080000:1.0:1775766231.915618:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766231.915623:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766231.915652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861387392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766231.915675:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766231.915689:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766231.915691:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766231.915694:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1960 00000100:00080000:1.0:1775766231.915698:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766231.915701:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766231.915710:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766231.915718:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861387392:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766231.915722:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861387520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766231.915732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766231.915735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766231.915736:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766231.915739:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1960 00000100:00080000:1.0:1775766231.915742:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766231.915743:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766231.915747:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766231.915756:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861387648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766231.915782:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766231.915823:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766231/real 1775766231] req@ffffa0e9c8ded500 x1862023861387392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766231.915838:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861387392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766231.915847:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766231.915851:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766231.915856:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861387392:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766231.915874:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766231.915892:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766231.915905:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766231/real 1775766231] req@ffffa0e9c8dec380 x1862023861387520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766231.915918:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861387520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766231.915927:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766231.915931:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766231.915934:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387520:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766231.915950:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766231.915974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766231.915987:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766231/real 1775766231] req@ffffa0e9c8deed80 x1862023861387648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766231.916001:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861387648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766231.916010:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766231.916028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766231.916032:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387648:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766232.491718:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766232.747597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766233.515610:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766233.771620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766234.539694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766234.795641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766235.563650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766235.819596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766236.587663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766236.843629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766237.035635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766237.035644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766237.035650:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766237.035661:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1965 00000100:00080000:1.0:1775766237.035674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766237.035696:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766237.035797:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861387776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766237.035852:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766237.035855:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766237.035857:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766237.035860:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1965 00000100:00080000:1.0:1775766237.035863:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766237.035866:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766237.035894:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766237.035908:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861387904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766237.035915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766237.035918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766237.035919:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766237.035921:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1965 00000100:00080000:1.0:1775766237.035923:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766237.035926:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766237.035929:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766237.035936:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861388032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766237.035963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861387776:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766237.036057:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766237.036109:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766237/real 1775766237] req@ffffa0e9c8def800 x1862023861387776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766237.036126:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861387776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766237.036135:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766237.036138:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766237.036142:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861387776:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766237.036147:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766237.036174:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766237.036182:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766237/real 1775766237] req@ffffa0e9c8def480 x1862023861387904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766237.036191:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861387904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766237.036196:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766237.036199:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766237.036201:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861387904:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766237.036204:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766237.036214:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766237.036219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766237/real 1775766237] req@ffffa0e9c8dedc00 x1862023861388032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766237.036226:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861388032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766237.036232:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766237.036234:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766237.036235:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388032:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766237.611634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766237.867596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766238.635673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766238.891642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766239.659668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766239.915596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766240.683709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766240.939629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766241.707701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766241.963577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766242.155618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766242.155629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766242.155635:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766242.155646:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1970 00000100:00080000:2.0:1775766242.155662:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766242.155669:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766242.155728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861388160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766242.155759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766242.155762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766242.155764:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766242.155768:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1970 00000100:00080000:2.0:1775766242.155793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766242.155798:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766242.155809:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766242.155820:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861388288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766242.155831:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766242.155834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766242.155836:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766242.155840:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1970 00000100:00080000:2.0:1775766242.155842:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766242.155845:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766242.155849:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766242.155861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861388416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766242.155862:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861388160:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766242.156012:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766242.156069:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766242/real 1775766242] req@ffffa0e9d0711500 x1862023861388160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766242.156103:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861388160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766242.156116:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766242.156121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766242.156125:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861388160:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766242.156134:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766242.156166:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766242.156176:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766242/real 1775766242] req@ffffa0e9d0713480 x1862023861388288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766242.156185:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861388288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766242.156191:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766242.156193:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766242.156195:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388288:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766242.156231:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766242.156242:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766242.156248:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766242/real 1775766242] req@ffffa0e9d0712a00 x1862023861388416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766242.156256:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861388416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766242.156271:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766242.156274:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766242.156276:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388416:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766242.731657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766242.987650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766243.755724:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766244.011629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766244.779628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766245.035565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766245.803570:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766246.059587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766246.827735:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766247.083667:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766247.275580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766247.275593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766247.275599:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766247.275609:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1975 00000100:00080000:2.0:1775766247.275621:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766247.275628:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766247.275687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861388544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766247.275718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766247.275721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766247.275723:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766247.275727:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1975 00000100:00080000:2.0:1775766247.275731:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766247.275734:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766247.275743:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766247.275754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861388672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766247.275764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766247.275768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766247.275770:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766247.275773:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1975 00000100:00080000:2.0:1775766247.275776:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766247.275780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766247.275802:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861388544:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775766247.275811:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766247.275822:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861388800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766247.275887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766247.275926:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766247/real 1775766247] req@ffffa0e9d0710e00 x1862023861388544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766247.275941:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861388544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766247.275948:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766247.275951:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766247.275954:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861388544:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766247.275960:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766247.275977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766247.275986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766247/real 1775766247] req@ffffa0e9d0710a80 x1862023861388672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766247.275993:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861388672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766247.275999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766247.276001:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766247.276012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388672:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766247.276034:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766247.276059:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766247.276067:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766247/real 1775766247] req@ffffa0e9d0712680 x1862023861388800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766247.276075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861388800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766247.276081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766247.276083:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766247.276085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861388800:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766247.851780:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766248.107556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766248.875636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766249.131609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766249.899658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766250.155593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766250.923722:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766251.179633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766251.947651:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766252.203606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766252.395588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766252.395601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766252.395606:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766252.395614:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1980 00000100:00080000:0.0:1775766252.395623:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766252.395628:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766252.395662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861388928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766252.395688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766252.395691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766252.395692:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766252.395708:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1980 00000100:00080000:0.0:1775766252.395711:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766252.395715:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766252.395723:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766252.395732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861389056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766252.395740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766252.395744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766252.395745:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766252.395748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1980 00000100:00080000:0.0:1775766252.395750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766252.395753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766252.395756:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766252.395761:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861389184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766252.395790:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861388928:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766252.395919:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766252.395977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766252/real 1775766252] req@ffffa0e9d09dce00 x1862023861388928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766252.396000:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861388928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766252.396014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766252.396019:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766252.396024:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861388928:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766252.396034:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766252.396060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766252.396075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766252/real 1775766252] req@ffffa0e9d09dc380 x1862023861389056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766252.396091:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861389056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766252.396100:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766252.396104:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766252.396108:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389056:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766252.396139:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766252.396163:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766252.396175:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766252/real 1775766252] req@ffffa0e9d09dc000 x1862023861389184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766252.396203:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861389184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766252.396212:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766252.396216:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766252.396220:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389184:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766252.971609:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766253.227606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766253.995753:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766254.251544:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766255.019655:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766255.275589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766256.043645:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766256.299607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766257.067640:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766257.323625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766257.515639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766257.515649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766257.515654:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766257.515665:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1986 00000100:00080000:2.0:1775766257.515675:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766257.515680:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766257.515715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861389312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766257.515739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766257.515742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766257.515743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766257.515745:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1986 00000100:00080000:2.0:1775766257.515748:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766257.515750:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766257.515758:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766257.515784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861389440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766257.515797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766257.515802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766257.515820:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766257.515825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1986 00000100:00080000:2.0:1775766257.515831:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766257.515839:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861389312:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766257.515841:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766257.515856:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766257.515880:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861389568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766257.515931:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766257.515989:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766257/real 1775766257] req@ffffa0e9d0712d80 x1862023861389312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766257.516011:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861389312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766257.516021:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766257.516025:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766257.516029:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861389312:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766257.516036:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766257.516055:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766257.516065:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766257/real 1775766257] req@ffffa0e9d0710700 x1862023861389440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766257.516074:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861389440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766257.516097:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766257.516102:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766257.516107:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389440:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766257.516140:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766257.516167:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766257.516178:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766257/real 1775766257] req@ffffa0e9d0711f80 x1862023861389568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766257.516188:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861389568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766257.516195:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766257.516198:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766257.516200:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389568:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766258.091620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766258.347611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766259.115682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766259.371589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766260.139720:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766260.395607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766261.163642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766261.419616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766262.187665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766262.443648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766262.635579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766262.635587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766262.635591:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766262.635601:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1991 00000100:00080000:2.0:1775766262.635610:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766262.635616:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766262.635652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861389696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766262.635677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766262.635679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766262.635681:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766262.635684:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1991 00000100:00080000:2.0:1775766262.635687:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766262.635706:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766262.635714:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766262.635741:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861389696:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766262.635743:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861389824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766262.635766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766262.635769:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766262.635771:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766262.635775:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1991 00000100:00080000:2.0:1775766262.635779:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766262.635783:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766262.635788:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766262.635797:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861389952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766262.635839:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766262.635887:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766262/real 1775766262] req@ffffa0e9d0711c00 x1862023861389696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766262.635919:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861389696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766262.635932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766262.635950:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766262.635954:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861389696:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766262.635983:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766262.636013:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766262.636023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766262/real 1775766262] req@ffffa0e9d0713b80 x1862023861389824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766262.636032:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861389824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766262.636038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766262.636040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766262.636043:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389824:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766262.636047:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766262.636058:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766262.636064:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766262/real 1775766262] req@ffffa0e9d0710000 x1862023861389952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766262.636071:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861389952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766262.636077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766262.636079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766262.636081:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861389952:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766263.211666:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766263.467564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766264.235688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766264.491605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766265.259746:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766265.515589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766266.283678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766266.539553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766267.307759:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766267.563587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766267.755639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766267.755661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766267.755667:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766267.755706:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 1996 00000100:00080000:1.0:1775766267.755724:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766267.755735:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766267.755800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861390080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766267.755850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766267.755856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766267.755859:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766267.755865:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1996 00000100:00080000:1.0:1775766267.755872:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766267.755879:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766267.755896:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766267.755915:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861390080:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766267.755924:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861390208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766267.755944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766267.755950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766267.755953:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766267.755959:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 1996 00000100:00080000:1.0:1775766267.755966:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766267.755972:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766267.755981:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766267.756009:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766267.756024:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861390336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766267.756066:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766267/real 1775766267] req@ffffa0e9c8deea00 x1862023861390080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766267.756105:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861390080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766267.756117:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766267.756143:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766267.756148:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861390080:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766267.756183:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766267.756215:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766267.756230:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766267/real 1775766267] req@ffffa0e9c8dee680 x1862023861390208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766267.756245:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861390208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766267.756256:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766267.756260:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766267.756264:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390208:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766267.756271:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766267.756289:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766267.756300:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766267/real 1775766267] req@ffffa0e9c8ded880 x1862023861390336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766267.756328:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861390336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766267.756347:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766267.756350:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766267.756353:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390336:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766268.331742:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766268.587682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766269.355773:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766269.611591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766270.379661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766270.635705:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766271.403620:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766271.659562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766272.427668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766272.683582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766272.875641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766272.875651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766272.875657:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766272.875666:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2001 00000100:00080000:0.0:1775766272.875677:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766272.875684:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766272.875721:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861390464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766272.875755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766272.875759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766272.875761:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766272.875765:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2001 00000100:00080000:0.0:1775766272.875770:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766272.875775:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766272.875784:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766272.875797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861390592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766272.875807:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861390464:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766272.875817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766272.875836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766272.875839:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766272.875845:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2001 00000100:00080000:0.0:1775766272.875850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766272.875854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766272.875861:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766272.875878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861390720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766272.875902:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766272.875952:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766272/real 1775766272] req@ffffa0e9d09df480 x1862023861390464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766272.875969:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861390464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766272.875979:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766272.875983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766272.875987:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861390464:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766272.876023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766272.876041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766272.876051:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766272/real 1775766272] req@ffffa0e9d09de300 x1862023861390592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766272.876060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861390592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766272.876066:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766272.876069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766272.876072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390592:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766272.876092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766272.876113:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766272.876125:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766272/real 1775766272] req@ffffa0e9d09dea00 x1862023861390720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766272.876140:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861390720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766272.876149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766272.876153:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766272.876157:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390720:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766273.451653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766273.707594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766274.475712:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766274.731563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766275.499719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766275.755623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766276.523722:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766276.779567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766277.547743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766277.803640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766277.995630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766277.995641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766277.995646:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766277.995657:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2006 00000100:00080000:0.0:1775766277.995670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766277.995677:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766277.995722:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861390848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766277.995750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766277.995753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766277.995755:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766277.995757:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2006 00000100:00080000:0.0:1775766277.995760:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766277.995762:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766277.995771:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766277.995799:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861390976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766277.995808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766277.995811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766277.995814:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766277.995814:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861390848:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766277.995818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2006 00000100:00080000:0.0:1775766277.995822:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766277.995827:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766277.995830:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766277.995838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861391104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766277.995892:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766277.995942:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766277/real 1775766277] req@ffffa0e9d09dea00 x1862023861390848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766277.995960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861390848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766277.995969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766277.995973:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766277.995978:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861390848:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766277.996003:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766277.996031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766277.996040:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766277/real 1775766277] req@ffffa0e9d09de300 x1862023861390976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766277.996048:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861390976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766277.996054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766277.996057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766277.996059:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861390976:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766277.996062:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766277.996073:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766277.996078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766277/real 1775766277] req@ffffa0e9d09df480 x1862023861391104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766277.996085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861391104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766277.996090:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766277.996092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766277.996094:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391104:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766278.571730:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766278.827564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766279.595714:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766279.851632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766280.619703:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766280.875610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766281.643746:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766281.899648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766282.667710:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766282.923597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766283.115700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766283.115715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766283.115723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766283.115739:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2011 00000100:00080000:1.0:1775766283.115801:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766283.115812:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766283.115869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861391232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766283.115917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766283.115922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766283.115924:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766283.115929:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2011 00000100:00080000:1.0:1775766283.115934:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766283.115939:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766283.115954:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766283.115971:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861391360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766283.115986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766283.115990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766283.115993:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766283.115997:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2011 00000100:00080000:1.0:1775766283.116001:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766283.116008:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766283.116016:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861391232:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766283.116017:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766283.116033:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861391488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766283.116156:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766283.116240:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766283/real 1775766283] req@ffffa0e9c8deca80 x1862023861391232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766283.116281:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861391232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766283.116304:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766283.116313:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766283.116321:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861391232:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766283.116338:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766283.116402:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766283.116429:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766283/real 1775766283] req@ffffa0e9c8dedf80 x1862023861391360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766283.116455:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861391360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766283.116490:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766283.116496:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766283.116557:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391360:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766283.116607:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766283.116643:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766283.116664:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766283/real 1775766283] req@ffffa0e9c8dece00 x1862023861391488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766283.116686:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861391488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766283.116702:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766283.116708:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766283.116715:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391488:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766283.691757:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766283.947709:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766284.715666:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766284.971570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766285.739619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766285.995621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766286.763688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766287.019577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766287.787635:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766288.043623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766288.235557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766288.235565:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766288.235569:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766288.235577:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2016 00000100:00080000:1.0:1775766288.235593:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766288.235599:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766288.235632:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861391616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766288.235655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766288.235657:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766288.235658:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766288.235661:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2016 00000100:00080000:1.0:1775766288.235664:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766288.235666:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766288.235674:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766288.235684:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861391744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766288.235695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766288.235698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766288.235699:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766288.235702:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2016 00000100:00080000:1.0:1775766288.235706:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766288.235713:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861391616:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766288.235713:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766288.235732:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766288.235744:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861391872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766288.235798:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766288.235853:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766288/real 1775766288] req@ffffa0e9c8defb80 x1862023861391616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766288.235904:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861391616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766288.235918:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766288.235924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766288.235930:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861391616:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766288.235963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766288.236002:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766288.236013:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766288/real 1775766288] req@ffffa0e9c8def100 x1862023861391744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766288.236022:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861391744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766288.236028:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766288.236031:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766288.236033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391744:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766288.236038:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766288.236050:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766288.236056:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766288/real 1775766288] req@ffffa0e9c8dec700 x1862023861391872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766288.236063:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861391872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766288.236078:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766288.236081:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766288.236083:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861391872:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766288.811761:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766289.067568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766289.835668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766290.091568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766290.859725:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766291.115641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766291.883637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766292.139579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766292.907714:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766293.163604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766293.355629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766293.355644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766293.355650:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766293.355664:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2021 00000100:00080000:1.0:1775766293.355680:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766293.355690:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766293.355761:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861392000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766293.355804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766293.355809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766293.355812:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766293.355817:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2021 00000100:00080000:1.0:1775766293.355823:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766293.355828:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766293.355858:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766293.355897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861392128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766293.355899:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861392000:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766293.355933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766293.355938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766293.355959:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766293.355966:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2021 00000100:00080000:1.0:1775766293.355981:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766293.355986:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766293.355993:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766293.356008:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861392256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766293.356041:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766293.356077:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392256:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766293.931629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766294.187588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766294.955700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766295.211689:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766295.979744:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766296.235692:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766296.427937:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766296.428006:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766296.428031:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766296.428101:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766296.428114:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766296.428173:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766293/real 1775766296] req@ffffa0e9c8def100 x1862023861392128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766296.428211:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861392128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766296.428227:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766296.428234:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766296.428240:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766296.428251:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766293/real 1775766296] req@ffffa0e9c8defb80 x1862023861392256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766296.428289:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861392256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766296.428301:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766296.428351:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766296.428355:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766296.428365:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766293/real 1775766296] req@ffffa0e9c8dec700 x1862023861392000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766296.428384:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861392000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766296.428395:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766296.428420:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766296.428426:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861392000:192.168.202.104@tcp:250:kworker.0 00010000:00010000:0.0:1775766297.003701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766297.259634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766298.027716:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766298.283623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766298.475582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766298.475594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766298.475600:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766298.475613:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2026 00000100:00080000:1.0:1775766298.475627:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766298.475635:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766298.475686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861392384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766298.475719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766298.475722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766298.475724:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766298.475728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2026 00000100:00080000:1.0:1775766298.475731:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766298.475734:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766298.475745:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766298.475754:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861392512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766298.475786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766298.475790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766298.475792:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766298.475796:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2026 00000100:00080000:1.0:1775766298.475799:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766298.475803:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766298.475808:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766298.475818:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861392384:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766298.475820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861392640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766298.475940:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766298.476035:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766298/real 1775766298] req@ffffa0e9c8dece00 x1862023861392384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766298.476066:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861392384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766298.476085:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766298.476092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766298.476098:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861392384:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766298.476125:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766298.476157:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766298.476174:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766298/real 1775766298] req@ffffa0e9c8dedf80 x1862023861392512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766298.476202:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861392512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766298.476212:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766298.476217:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766298.476221:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392512:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766298.476263:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766298.476314:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766298.476330:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766298/real 1775766298] req@ffffa0e9c8deca80 x1862023861392640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766298.476346:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861392640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766298.476355:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766298.476360:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766298.476363:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392640:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766299.051709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766299.307657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766300.075708:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766300.331634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766301.099639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766301.355644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766302.123656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766302.379600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766303.147752:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766303.403597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766303.595656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766303.595669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766303.595676:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766303.595705:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2032 00000100:00080000:1.0:1775766303.595721:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766303.595731:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766303.595789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861392768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766303.595834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766303.595839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766303.595841:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766303.595845:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2032 00000100:00080000:1.0:1775766303.595873:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766303.595878:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766303.595891:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766303.595908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861392896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766303.595920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766303.595924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766303.595926:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766303.595930:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2032 00000100:00080000:1.0:1775766303.595934:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766303.595941:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766303.595944:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861392768:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766303.595947:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766303.595964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861393024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766303.596058:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766303.596123:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766303/real 1775766303] req@ffffa0e9c8ded880 x1862023861392768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766303.596147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861392768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766303.596169:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766303.596174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766303.596179:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861392768:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766303.596187:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766303.596231:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766303.596245:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766303/real 1775766303] req@ffffa0e9c8dee680 x1862023861392896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766303.596258:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861392896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766303.596267:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766303.596270:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766303.596273:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861392896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766303.596303:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766303.596321:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766303.596353:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766303/real 1775766303] req@ffffa0e9c8deea00 x1862023861393024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766303.596388:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861393024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766303.596396:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766303.596399:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766303.596402:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393024:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766304.171608:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766304.427579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766305.195676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766305.451593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766306.219652:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766306.475637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766307.243595:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766307.499621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766308.267637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766308.523604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766308.715594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766308.715603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766308.715607:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766308.715615:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2037 00000100:00080000:3.0:1775766308.715624:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766308.715630:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766308.715666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861393152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766308.715688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766308.715690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766308.715696:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766308.715699:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2037 00000100:00080000:3.0:1775766308.715702:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766308.715704:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766308.715713:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766308.715746:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861393152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766308.715751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861393280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766308.715760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766308.715763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766308.715764:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766308.715767:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2037 00000100:00080000:3.0:1775766308.715771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766308.715774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766308.715778:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766308.715796:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861393408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766308.715815:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766308.715866:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766308/real 1775766308] req@ffffa0e9d0542680 x1862023861393152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766308.715882:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861393152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766308.715891:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766308.715894:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766308.715897:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861393152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766308.715915:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766308.715930:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766308.715947:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766308/real 1775766308] req@ffffa0e9d0541880 x1862023861393280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766308.715955:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861393280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766308.715960:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766308.715971:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766308.715973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393280:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766308.715977:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766308.715989:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766308.716003:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766308/real 1775766308] req@ffffa0e9d0541500 x1862023861393408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766308.716011:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861393408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766308.716017:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766308.716019:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766308.716021:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393408:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766309.291704:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766309.547615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766310.315770:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766310.571629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766311.339649:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766311.595636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766312.363750:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766312.619651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766313.387688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766313.643572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766313.835589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766313.835598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766313.835602:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766313.835610:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2042 00000100:00080000:3.0:1775766313.835619:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766313.835624:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766313.835658:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861393536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766313.835682:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766313.835685:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766313.835686:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766313.835704:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2042 00000100:00080000:3.0:1775766313.835707:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766313.835710:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766313.835717:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766313.835726:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861393664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766313.835733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766313.835736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766313.835737:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766313.835739:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2042 00000100:00080000:3.0:1775766313.835741:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766313.835744:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766313.835747:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766313.835757:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861393792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766313.835796:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861393536:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766313.835889:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766313.835942:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766313/real 1775766313] req@ffffa0e9d0542a00 x1862023861393536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766313.835963:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861393536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766313.835974:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766313.835978:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766313.835982:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861393536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766313.835990:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766313.836009:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766313.836019:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766313/real 1775766313] req@ffffa0e9d0540000 x1862023861393664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766313.836051:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861393664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766313.836059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766313.836062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766313.836065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766313.836070:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766313.836087:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766313.836095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766313/real 1775766313] req@ffffa0e9d0543b80 x1862023861393792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766313.836105:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861393792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766313.836111:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766313.836113:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766313.836116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861393792:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766314.411744:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766314.667583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766315.435770:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766315.691566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766316.459623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766316.715651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766317.483698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766317.739565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766318.507690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766318.763649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766318.955672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766318.955685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766318.955692:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766318.955704:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2047 00000100:00080000:2.0:1775766318.955719:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766318.955728:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766318.955785:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861393920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766318.955832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766318.955836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766318.955837:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766318.955841:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2047 00000100:00080000:2.0:1775766318.955844:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766318.955873:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766318.955885:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766318.955898:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861394048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766318.955908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766318.955912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766318.955914:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766318.955918:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2047 00000100:00080000:2.0:1775766318.955922:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766318.955925:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766318.955929:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766318.955942:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861394176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766318.955943:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861393920:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766318.956040:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766318.956112:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766318/real 1775766318] req@ffffa0e9d0710380 x1862023861393920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766318.956136:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861393920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766318.956160:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766318.956165:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766318.956170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861393920:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766318.956178:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766318.956250:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766318.956268:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766318/real 1775766318] req@ffffa0e9d0711180 x1862023861394048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766318.956287:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861394048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766318.956299:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766318.956304:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766318.956308:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766318.956342:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766318.956362:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766318.956372:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766318/real 1775766318] req@ffffa0e9d0710000 x1862023861394176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766318.956382:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861394176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766318.956389:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766318.956392:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766318.956395:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394176:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766319.531603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766319.787586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766320.555716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766320.811662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766321.579665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766321.835645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766322.603728:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766322.859581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766323.627636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766323.883661:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766324.075745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766324.075761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766324.075769:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766324.075785:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2052 00000100:00080000:1.0:1775766324.075836:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766324.075847:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766324.075923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861394304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766324.075974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766324.075980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766324.075982:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766324.075988:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2052 00000100:00080000:1.0:1775766324.076002:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766324.076006:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766324.076022:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766324.076037:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861394432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766324.076050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766324.076054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766324.076056:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766324.076060:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2052 00000100:00080000:1.0:1775766324.076064:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766324.076078:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766324.076082:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766324.076092:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861394560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766324.076186:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861394304:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766324.076288:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766324.076370:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766324/real 1775766324] req@ffffa0e9c8dedc00 x1862023861394304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766324.076411:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861394304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766324.076427:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766324.076433:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766324.076439:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861394304:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766324.076449:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766324.076477:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766324.076557:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766324/real 1775766324] req@ffffa0e9c8def480 x1862023861394432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766324.076575:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861394432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766324.076586:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766324.076591:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766324.076595:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394432:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766324.076603:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766324.076628:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766324.076657:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766324/real 1775766324] req@ffffa0e9c8def800 x1862023861394560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766324.076673:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861394560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766324.076683:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766324.076687:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766324.076691:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394560:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766324.651775:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766324.907622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766325.675726:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766325.931583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766326.699724:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766326.955566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766327.723689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766327.979606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766328.747580:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766329.003601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766329.195564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766329.195578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766329.195584:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766329.195596:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2057 00000100:00080000:1.0:1775766329.195612:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766329.195621:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766329.195682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861394688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766329.195729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766329.195733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766329.195734:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766329.195738:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2057 00000100:00080000:1.0:1775766329.195742:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766329.195745:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766329.195756:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766329.195767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861394816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766329.195775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766329.195778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766329.195780:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766329.195782:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2057 00000100:00080000:1.0:1775766329.195784:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766329.195787:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766329.195790:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766329.195835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861394944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766329.195878:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861394688:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766329.195949:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766329.195991:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766329/real 1775766329] req@ffffa0e9c8deed80 x1862023861394688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766329.196007:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861394688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766329.196016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766329.196020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766329.196023:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861394688:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766329.196029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766329.196044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766329.196052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766329/real 1775766329] req@ffffa0e9c8dec380 x1862023861394816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766329.196080:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861394816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766329.196103:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766329.196122:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766329.196125:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394816:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766329.196147:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766329.196166:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766329.196176:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766329/real 1775766329] req@ffffa0e9c8ded500 x1862023861394944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766329.196185:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861394944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766329.196192:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766329.196194:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766329.196197:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861394944:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766329.771677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766330.027619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766330.795739:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766331.051609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766331.819667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766332.075651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766332.843682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766333.099574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766333.867719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766334.123628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766334.315600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766334.315611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766334.315616:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766334.315628:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2062 00000100:00080000:1.0:1775766334.315640:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766334.315647:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766334.315695:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861395072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766334.315725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766334.315728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766334.315730:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766334.315733:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2062 00000100:00080000:1.0:1775766334.315736:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766334.315738:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766334.315748:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766334.315757:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861395200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766334.315764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766334.315767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766334.315768:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766334.315790:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2062 00000100:00080000:1.0:1775766334.315794:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766334.315798:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766334.315802:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766334.315810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861395328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766334.315820:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861395072:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766334.315893:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766334.315946:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766334/real 1775766334] req@ffffa0e9c8dee300 x1862023861395072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766334.315966:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861395072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766334.315983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766334.315987:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766334.315991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861395072:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766334.315997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766334.316016:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766334.316025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766334/real 1775766334] req@ffffa0e9c8dec000 x1862023861395200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766334.316034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861395200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766334.316059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766334.316062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766334.316064:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766334.316087:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766334.316103:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766334.316111:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766334/real 1775766334] req@ffffa0e9c8ded180 x1862023861395328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766334.316119:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861395328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766334.316125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766334.316127:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766334.316129:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395328:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766334.891650:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766335.147580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766335.915682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766336.171655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766336.939629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766337.195601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766337.963613:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766338.219611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766338.987648:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766339.243624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766339.435710:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766339.435726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766339.435734:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766339.435749:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2067 00000100:00080000:0.0:1775766339.435780:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766339.435792:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766339.435854:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861395456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766339.435901:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766339.435907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766339.435910:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766339.435917:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2067 00000100:00080000:0.0:1775766339.435923:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766339.435930:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766339.435945:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861395456:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775766339.435946:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766339.435974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861395584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766339.436010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766339.436016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766339.436045:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766339.436052:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2067 00000100:00080000:0.0:1775766339.436071:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766339.436078:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:1.0:1775766339.436084:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:0.0:1775766339.436111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766339.436133:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861395712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766339.436176:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766339/real 1775766339] req@ffffa0e9d09dc000 x1862023861395456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766339.436209:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861395456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766339.436221:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766339.436226:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766339.436231:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861395456:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766339.436258:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766339.436283:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766339.436295:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766339/real 1775766339] req@ffffa0e9d09dc380 x1862023861395584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766339.436306:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861395584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766339.436328:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766339.436331:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766339.436335:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395584:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766339.436340:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766339.436358:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766339.436367:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766339/real 1775766339] req@ffffa0e9d09dce00 x1862023861395712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766339.436377:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861395712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766339.436383:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766339.436385:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766339.436388:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395712:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766340.011673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766340.267566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766341.035603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766341.291606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766342.059681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766342.315629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766343.083693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766343.339598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766344.107622:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766344.363597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766344.555579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766344.555590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766344.555596:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766344.555606:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2073 00000100:00080000:0.0:1775766344.555618:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766344.555625:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766344.555674:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861395840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766344.555707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766344.555709:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766344.555711:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766344.555714:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2073 00000100:00080000:0.0:1775766344.555716:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766344.555719:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766344.555736:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766344.555746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861395968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766344.555754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766344.555757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775766344.555773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861395840:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766344.555785:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766344.555790:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2073 00000100:00080000:0.0:1775766344.555793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766344.555796:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766344.555801:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766344.555814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861396096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766344.555867:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766344.555921:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766344/real 1775766344] req@ffffa0e9d09df100 x1862023861395840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766344.555956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861395840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766344.555965:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766344.555969:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766344.555979:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861395840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766344.555992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766344.556013:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766344.556023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766344/real 1775766344] req@ffffa0e9d09dfb80 x1862023861395968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766344.556032:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861395968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766344.556038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766344.556040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766344.556042:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861395968:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766344.556065:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766344.556076:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766344.556082:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766344/real 1775766344] req@ffffa0e9d09ded80 x1862023861396096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766344.556090:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861396096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766344.556095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766344.556097:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766344.556098:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396096:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766345.131718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766345.387595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766346.155653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766346.411604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766347.179654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766347.435572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766348.203644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766348.459582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766349.227693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766349.483635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766349.675643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766349.675653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766349.675658:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766349.675676:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2078 00000100:00080000:3.0:1775766349.675704:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766349.675711:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766349.675757:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861396224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766349.675799:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766349.675804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766349.675806:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766349.675811:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2078 00000100:00080000:3.0:1775766349.675816:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766349.675821:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766349.675833:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766349.675848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861396352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766349.675857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766349.675860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766349.675861:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766349.675864:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2078 00000100:00080000:3.0:1775766349.675867:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766349.675870:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766349.675873:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766349.675880:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861396224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766349.675928:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861396480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766349.675991:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766349.676044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766349/real 1775766349] req@ffffa0e9d0541c00 x1862023861396224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766349.676061:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861396224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766349.676070:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766349.676073:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766349.676077:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861396224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766349.676082:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766349.676098:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766349.676106:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766349/real 1775766349] req@ffffa0e9d0540380 x1862023861396352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766349.676114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861396352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766349.676120:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766349.676122:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766349.676124:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396352:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766349.676146:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766349.676156:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766349.676161:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766349/real 1775766349] req@ffffa0e9d0542300 x1862023861396480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766349.676181:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861396480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766349.676187:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766349.676189:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766349.676191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396480:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766350.251616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766350.507624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766351.275622:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766351.531649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766352.299664:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766352.555589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766353.323657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766353.579619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766354.347685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766354.603560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766354.795641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766354.795652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766354.795657:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766354.795667:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2083 00000100:00080000:1.0:1775766354.795679:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766354.795684:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766354.795718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861396608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766354.795748:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766354.795752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766354.795753:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766354.795758:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2083 00000100:00080000:1.0:1775766354.795762:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766354.795766:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766354.795777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766354.795802:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861396608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766354.795805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861396736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766354.795817:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766354.795819:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766354.795821:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766354.795824:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2083 00000100:00080000:1.0:1775766354.795828:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766354.795831:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766354.795837:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766354.795867:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861396864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766354.795910:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766354.795960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766354/real 1775766354] req@ffffa0e9c8ded180 x1862023861396608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766354.795975:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861396608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766354.795984:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766354.795988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766354.796002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861396608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766354.796024:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766354.796049:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766354.796063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766354/real 1775766354] req@ffffa0e9c8dec000 x1862023861396736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766354.796087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861396736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766354.796094:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766354.796096:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766354.796098:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396736:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766354.796103:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766354.796118:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766354.796125:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766354/real 1775766354] req@ffffa0e9c8dee300 x1862023861396864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766354.796132:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861396864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766354.796137:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766354.796139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766354.796141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861396864:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766355.371763:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766355.627659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766356.395646:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766356.651634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766357.419693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766357.675572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766358.443651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766358.699604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766359.467642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766359.723590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766359.915608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766359.915619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766359.915625:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766359.915646:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2088 00000100:00080000:0.0:1775766359.915657:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766359.915664:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766359.915729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861396992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766359.915762:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766359.915765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766359.915766:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766359.915770:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2088 00000100:00080000:0.0:1775766359.915774:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766359.915777:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766359.915788:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766359.915813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861397120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766359.915833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766359.915836:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766359.915837:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766359.915843:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2088 00000100:00080000:0.0:1775766359.915847:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766359.915853:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766359.915854:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861396992:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775766359.915857:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766359.915878:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861397248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766359.915987:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397120:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766359.916019:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397248:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766360.491672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766360.747625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766361.515714:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766361.771627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766362.539654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766362.795594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775766362.987798:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775766362.987847:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766362.987865:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766362.987914:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766362.987924:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775766362.987986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766359/real 1775766362] req@ffffa0e9d09dc700 x1862023861397120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766362.988019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861397120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766362.988034:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766362.988039:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766362.988063:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397120:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766362.988073:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766359/real 1775766362] req@ffffa0e9d09df800 x1862023861397248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766362.988087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861397248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766362.988095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766362.988099:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766362.988103:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397248:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766362.988111:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766359/real 1775766362] req@ffffa0e9d09dd880 x1862023861396992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766362.988124:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861396992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766362.988132:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766362.988136:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766362.988140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861396992:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775766363.563615:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766363.819656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766364.587803:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766364.843580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766365.035862:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766365.035877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766365.035884:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766365.035898:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2093 00000100:00080000:1.0:1775766365.035915:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766365.035926:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766365.036034:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861397376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766365.036171:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861397376:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766365.036231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766365.036238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766365.036241:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766365.036249:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2093 00000100:00080000:1.0:1775766365.036277:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766365.036285:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775766365.036295:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775766365.036305:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766365.036329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861397504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766365.036346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766365.036352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766365.036355:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766365.036363:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2093 00000100:00100000:0.0:1775766365.036365:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766365/real 1775766365] req@ffffa0e9c8ded500 x1862023861397376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766365.036370:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766365.036380:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766365.036394:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766365.036411:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861397376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766365.036414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861397632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766365.036448:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766365.036477:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766365.036516:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861397376:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766365.036557:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766365.036589:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766365.036606:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766365/real 1775766365] req@ffffa0e9c8dec380 x1862023861397504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766365.036621:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861397504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766365.036631:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766365.036635:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766365.036639:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397504:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766365.036647:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766365.036682:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766365.036696:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766365/real 1775766365] req@ffffa0e9c8deed80 x1862023861397632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766365.036740:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861397632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766365.036768:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766365.036773:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766365.036778:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397632:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766365.611627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766365.867609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766366.635722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766366.891653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766367.659651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766367.915663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766368.683698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766368.939627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766369.707684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766369.963654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766370.155599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766370.155638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766370.155644:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766370.155658:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2098 00000100:00080000:1.0:1775766370.155674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766370.155684:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766370.155761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861397760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766370.155813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766370.155818:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766370.155820:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766370.155825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2098 00000100:00080000:1.0:1775766370.155830:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766370.155835:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766370.155849:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766370.155868:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861397888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766370.155882:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766370.155886:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766370.155888:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766370.155894:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2098 00000100:00080000:1.0:1775766370.155911:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766370.155916:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766370.155922:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766370.155934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861397760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766370.155939:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861398016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766370.156015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766370.156067:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766370/real 1775766370] req@ffffa0e9c8def800 x1862023861397760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766370.156097:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861397760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766370.156128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766370.156134:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766370.156140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861397760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766370.156151:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766370.156182:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766370.156198:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766370/real 1775766370] req@ffffa0e9c8def480 x1862023861397888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766370.156228:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861397888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766370.156254:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766370.156257:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766370.156260:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861397888:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766370.156305:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766370.156331:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766370.156348:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766370/real 1775766370] req@ffffa0e9c8dedc00 x1862023861398016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766370.156372:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861398016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766370.156387:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766370.156408:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766370.156413:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398016:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766370.731671:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766370.987625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766371.755787:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766372.011573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766372.779803:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766373.035677:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766373.803762:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766374.059622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766374.827691:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766375.083628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766375.275583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766375.275591:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766375.275595:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766375.275603:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2103 00000100:00080000:0.0:1775766375.275612:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766375.275617:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766375.275664:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861398144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766375.275716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766375.275718:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766375.275720:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766375.275723:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2103 00000100:00080000:0.0:1775766375.275726:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766375.275730:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766375.275738:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766375.275748:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861398272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766375.275756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766375.275759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775766375.275760:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861398144:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766375.275761:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766375.275768:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2103 00000100:00080000:0.0:1775766375.275772:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766375.275774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766375.275777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766375.275785:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861398400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766375.275834:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766375.275873:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766375/real 1775766375] req@ffffa0e9d09dd500 x1862023861398144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766375.275890:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861398144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766375.275897:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766375.275900:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766375.275902:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861398144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766375.275921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766375.275937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766375.275945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766375/real 1775766375] req@ffffa0e9d09dd180 x1862023861398272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766375.275953:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861398272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766375.275958:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766375.275960:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766375.275963:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766375.275966:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766375.275987:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766375.275994:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766375/real 1775766375] req@ffffa0e9d09ddc00 x1862023861398400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766375.276002:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861398400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766375.276008:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766375.276010:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766375.276012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398400:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766375.851775:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766376.107598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766376.875755:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766377.131611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766377.899692:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766378.155680:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766378.923814:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766379.179568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766379.947720:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766380.203573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766380.395635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766380.395647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766380.395654:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766380.395667:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2108 00000100:00080000:2.0:1775766380.395683:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766380.395692:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766380.395745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861398528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766380.395776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766380.395780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766380.395782:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766380.395786:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2108 00000100:00080000:2.0:1775766380.395789:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766380.395793:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766380.395806:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766380.395843:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861398528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766380.395854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861398656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766380.395878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766380.395882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766380.395884:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766380.395889:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2108 00000100:00080000:2.0:1775766380.395893:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766380.395896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766380.395902:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766380.395915:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861398784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766380.395972:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766380.396044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766380/real 1775766380] req@ffffa0e9d0713b80 x1862023861398528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766380.396082:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861398528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766380.396099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766380.396106:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766380.396112:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861398528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766380.396148:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766380.396175:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766380.396188:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766380/real 1775766380] req@ffffa0e9d0711c00 x1862023861398656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766380.396210:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861398656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766380.396219:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766380.396222:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766380.396225:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398656:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766380.396231:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766380.396257:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766380.396272:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766380/real 1775766380] req@ffffa0e9d0711f80 x1862023861398784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766380.396306:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861398784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766380.396319:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766380.396325:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766380.396330:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861398784:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766380.971574:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766381.227622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766381.995695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766382.251596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766383.019717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766383.275696:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766384.043686:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766384.299557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766385.067694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766385.323630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766385.515646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766385.515658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766385.515664:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766385.515676:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2114 00000100:00080000:1.0:1775766385.515691:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766385.515707:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766385.515780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861398912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766385.515815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766385.515818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766385.515820:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766385.515823:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2114 00000100:00080000:1.0:1775766385.515827:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766385.515831:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766385.515843:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766385.515854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861399040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766385.515863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766385.515867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766385.515868:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766385.515871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2114 00000100:00080000:1.0:1775766385.515874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766385.515879:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766385.515883:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861398912:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766385.515884:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766385.515896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861399168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766385.515984:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766385.516066:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766385/real 1775766385] req@ffffa0e9c8deea00 x1862023861398912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.516098:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861398912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.516114:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.516120:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766385.516127:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861398912:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766385.516138:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766385.516169:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766385.516187:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766385/real 1775766385] req@ffffa0e9c8dee680 x1862023861399040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.516204:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861399040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.516215:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.516220:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766385.516225:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766385.516259:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766385.516306:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766385.516324:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766385/real 1775766385] req@ffffa0e9c8ded880 x1862023861399168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.516345:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861399168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.516378:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.516385:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766385.516390:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399168:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766386.091700:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766386.347560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766387.115731:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766387.371626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766388.139800:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766388.395654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766389.163696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766389.419585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766390.187663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766390.443601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766390.635703:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766390.635714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766390.635720:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766390.635731:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2119 00000100:00080000:0.0:1775766390.635765:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766390.635773:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766390.635844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861399296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766390.635880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766390.635884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766390.635885:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766390.635889:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2119 00000100:00080000:0.0:1775766390.635892:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766390.635896:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766390.635909:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766390.635921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861399424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766390.635931:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861399296:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766390.635934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766390.635963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766390.635965:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766390.635970:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2119 00000100:00080000:0.0:1775766390.635975:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766390.635979:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766390.635984:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766390.635995:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861399552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766390.636026:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766390.636106:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766390/real 1775766390] req@ffffa0e9d09ddf80 x1862023861399296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766390.636124:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861399296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766390.636133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766390.636138:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766390.636141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861399296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766390.636184:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766390.636206:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766390.636217:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766390/real 1775766390] req@ffffa0e9d09dca80 x1862023861399424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766390.636227:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861399424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766390.636234:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766390.636237:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766390.636239:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399424:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766390.636243:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766390.636257:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766390.636264:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766390/real 1775766390] req@ffffa0e9d09de680 x1862023861399552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766390.636274:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861399552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766390.636293:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766390.636296:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766390.636298:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399552:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766391.211739:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766391.467648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766392.235725:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766392.491612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766393.259628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766393.515608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766394.283694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766394.539682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766395.307794:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766395.563699:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766395.755724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766395.755762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766395.755768:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766395.755785:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2124 00000100:00080000:2.0:1775766395.755804:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766395.755816:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766395.755873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861399680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766395.755938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766395.755946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766395.755950:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766395.755959:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2124 00000100:00080000:2.0:1775766395.755967:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766395.755976:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766395.755993:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766395.756022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861399808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766395.756040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861399680:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766395.756053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766395.756061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766395.756065:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766395.756075:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2124 00000100:00080000:2.0:1775766395.756083:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766395.756093:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766395.756104:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766395.756132:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861399936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766395.756153:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766395.756229:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766395/real 1775766395] req@ffffa0e9d0710700 x1862023861399680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766395.756260:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861399680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766395.756275:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766395.756280:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766395.756286:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861399680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766395.756315:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766395.756343:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766395.756358:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766395/real 1775766395] req@ffffa0e9d0712d80 x1862023861399808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766395.756373:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861399808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766395.756383:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766395.756400:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766395.756405:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399808:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766395.756413:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766395.756439:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766395.756477:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766395/real 1775766395] req@ffffa0e9d0712680 x1862023861399936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766395.756540:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861399936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766395.756550:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766395.756554:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766395.756559:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861399936:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766396.331685:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766396.587621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766397.355732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766397.611615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766398.379724:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766398.635582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766399.403677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766399.659616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766400.427667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766400.683575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766400.875619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766400.875632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766400.875638:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766400.875650:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2129 00000100:00080000:2.0:1775766400.875664:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766400.875673:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766400.875721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861400064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766400.875761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766400.875766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766400.875769:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766400.875774:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2129 00000100:00080000:2.0:1775766400.875778:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766400.875782:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766400.875794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766400.875808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861400192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766400.875823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766400.875828:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766400.875851:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766400.875857:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2129 00000100:00080000:2.0:1775766400.875864:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766400.875868:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861400064:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766400.875872:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766400.875884:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766400.875903:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861400320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766400.875980:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766400.876034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766400/real 1775766400] req@ffffa0e9d0710a80 x1862023861400064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766400.876058:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861400064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766400.876072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766400.876077:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766400.876083:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861400064:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766400.876105:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766400.876135:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766400.876150:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766400/real 1775766400] req@ffffa0e9d0710e00 x1862023861400192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766400.876166:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861400192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766400.876176:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766400.876180:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766400.876184:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766400.876214:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766400.876234:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766400.876245:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766400/real 1775766400] req@ffffa0e9d0712a00 x1862023861400320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766400.876259:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861400320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766400.876269:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766400.876272:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766400.876276:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400320:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766401.451785:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766401.707574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766402.475695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766402.731553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766403.499763:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766403.755636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766404.523665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766404.779622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766405.547671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766405.803648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766405.995708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766405.995724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766405.995763:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766405.995778:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2134 00000100:00080000:1.0:1775766405.995804:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766405.995815:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766405.995882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861400448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766405.995936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766405.995942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766405.995944:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766405.995949:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2134 00000100:00080000:1.0:1775766405.995954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766405.995958:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766405.995973:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766405.995991:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861400576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766405.996006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766405.996011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766405.996014:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766405.996022:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2134 00000100:00100000:0.0:1775766405.996024:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861400448:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766405.996026:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766405.996038:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766405.996046:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766405.996061:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861400704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766405.996156:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766405.996228:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766405/real 1775766405] req@ffffa0e9c8deca80 x1862023861400448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766405.996270:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861400448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766405.996286:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766405.996292:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766405.996297:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861400448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766405.996330:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766405.996389:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766405.996421:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766405/real 1775766405] req@ffffa0e9c8dedf80 x1862023861400576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766405.996440:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861400576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766405.996451:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766405.996455:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766405.996460:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400576:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766405.996471:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766405.996614:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766405.996634:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766405/real 1775766405] req@ffffa0e9c8dece00 x1862023861400704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766405.996651:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861400704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766405.996661:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766405.996666:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766405.996679:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400704:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766406.571630:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766406.827569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766407.595703:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766407.851645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766408.619616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766408.875605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766409.643690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766409.899552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766410.667669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766410.923606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766411.115673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766411.115693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766411.115703:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766411.115722:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2139 00000100:00080000:1.0:1775766411.115743:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766411.115756:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766411.115821:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861400832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766411.115867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766411.115872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766411.115876:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766411.115880:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2139 00000100:00080000:1.0:1775766411.115886:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766411.115892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766411.115918:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766411.115937:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861400960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766411.115956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861400832:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766411.115972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766411.115979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766411.115985:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766411.115998:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2139 00000100:00080000:1.0:1775766411.116032:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766411.116051:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766411.116060:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766411.116078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861401088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766411.116118:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766411.116176:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766411/real 1775766411] req@ffffa0e9c8dec700 x1862023861400832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766411.116217:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861400832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766411.116228:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766411.116232:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766411.116236:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861400832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766411.116261:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766411.116284:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766411.116337:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766411/real 1775766411] req@ffffa0e9c8defb80 x1862023861400960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766411.116348:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861400960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766411.116355:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766411.116358:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766411.116360:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861400960:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766411.116366:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766411.116384:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766411.116391:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766411/real 1775766411] req@ffffa0e9c8def100 x1862023861401088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766411.116405:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861401088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766411.116441:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766411.116447:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766411.116451:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401088:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766411.691639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766411.947575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766412.715596:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766412.971574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766413.739759:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766413.995701:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766414.763587:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766415.019608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766415.787751:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766416.043630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766416.235588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766416.235597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766416.235601:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766416.235610:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2144 00000100:00080000:1.0:1775766416.235621:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766416.235629:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766416.235689:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861401216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766416.235723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766416.235726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766416.235728:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766416.235731:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2144 00000100:00080000:1.0:1775766416.235735:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766416.235738:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766416.235748:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766416.235759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861401344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766416.235813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766416.235816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766416.235817:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766416.235821:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2144 00000100:00080000:1.0:1775766416.235824:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766416.235827:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766416.235831:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766416.235839:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861401472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766416.235861:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861401216:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766416.235934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766416.235977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766416/real 1775766416] req@ffffa0e9c8def100 x1862023861401216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766416.235995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861401216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766416.236003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766416.236006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766416.236009:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861401216:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766416.236014:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766416.236030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766416.236050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766416/real 1775766416] req@ffffa0e9c8defb80 x1862023861401344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766416.236058:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861401344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766416.236063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766416.236065:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766416.236075:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401344:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766416.236093:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766416.236107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766416.236123:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766416/real 1775766416] req@ffffa0e9c8dec700 x1862023861401472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766416.236131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861401472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766416.236137:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766416.236139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766416.236141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401472:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766416.811642:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766417.067561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766417.835614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766418.091604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766418.859761:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766419.115589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766419.883655:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766420.139590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766420.907581:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766421.163567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766421.355626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766421.355637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766421.355643:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766421.355655:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2149 00000100:00080000:1.0:1775766421.355670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766421.355677:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766421.355732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861401600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766421.355766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766421.355780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766421.355782:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766421.355787:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2149 00000100:00080000:1.0:1775766421.355790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766421.355812:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766421.355824:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766421.355838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861401728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766421.355850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766421.355854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766421.355857:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766421.355862:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2149 00000100:00080000:1.0:1775766421.355867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766421.355871:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861401600:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766421.355874:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766421.355888:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766421.355904:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861401856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766421.355951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766421.355991:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766421/real 1775766421] req@ffffa0e9c8dece00 x1862023861401600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766421.356006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861401600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766421.356014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766421.356017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766421.356020:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861401600:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766421.356026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766421.356056:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766421.356068:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766421/real 1775766421] req@ffffa0e9c8dedf80 x1862023861401728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766421.356081:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861401728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766421.356089:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766421.356093:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766421.356096:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401728:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766421.356121:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766421.356140:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766421.356158:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766421/real 1775766421] req@ffffa0e9c8deca80 x1862023861401856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766421.356166:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861401856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766421.356172:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766421.356174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766421.356176:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861401856:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766421.931657:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766422.187560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766422.955642:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766423.211552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766423.979656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766424.235620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766425.003673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766425.259647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766426.027784:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766426.283596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766426.475616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766426.475629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766426.475635:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766426.475645:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2154 00000100:00080000:1.0:1775766426.475656:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766426.475664:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766426.475706:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861401984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766426.475732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766426.475754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766426.475756:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766426.475760:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2154 00000100:00080000:1.0:1775766426.475764:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766426.475767:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766426.475777:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766426.475787:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861402112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766426.475826:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766426.475831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766426.475834:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766426.475841:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2154 00000100:00080000:1.0:1775766426.475849:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766426.475854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766426.475862:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766426.475895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861402240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766426.475916:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861401984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766426.476163:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402112:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766426.476277:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402240:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766427.051605:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766427.307579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766428.075635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766428.331576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766429.099675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766429.355623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766429.547737:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766429.547774:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766429.547788:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766429.547819:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766429.547828:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766429.547859:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766426/real 1775766429] req@ffffa0e9c8deea00 x1862023861402240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766429.547884:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861402240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766429.547896:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766429.547901:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766429.547906:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402240:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766429.547914:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766426/real 1775766429] req@ffffa0e9c8ded880 x1862023861401984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766429.547929:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861401984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766429.547936:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766429.547940:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766429.547943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861401984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766429.547949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766426/real 1775766429] req@ffffa0e9c8dee680 x1862023861402112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766429.547961:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861402112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766429.547981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766429.548009:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766429.548013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402112:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766430.123591:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766430.379547:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766431.147798:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766431.403606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766431.595635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766431.595644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766431.595648:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766431.595657:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2160 00000100:00080000:2.0:1775766431.595706:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766431.595720:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766431.595759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861402368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766431.595816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766431.595821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766431.595823:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766431.595828:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2160 00000100:00080000:2.0:1775766431.595833:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766431.595838:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766431.595869:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766431.595886:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861402496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766431.595900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766431.595903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766431.595906:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766431.595910:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2160 00000100:00080000:2.0:1775766431.595925:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766431.595930:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766431.595936:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766431.595947:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861402624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766431.596004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861402368:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766431.596109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766431.596181:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766431/real 1775766431] req@ffffa0e9d0713480 x1862023861402368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766431.596209:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861402368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766431.596224:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766431.596229:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766431.596234:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861402368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766431.596243:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766431.596271:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766431.596295:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766431/real 1775766431] req@ffffa0e9d0711500 x1862023861402496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766431.596312:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861402496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766431.596322:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766431.596327:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766431.596330:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402496:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766431.596337:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766431.596374:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766431.596390:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766431/real 1775766431] req@ffffa0e9d0711880 x1862023861402624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766431.596407:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861402624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766431.596417:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766431.596421:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766431.596425:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402624:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766432.171642:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766432.427569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766433.195657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766433.451564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766434.219648:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766434.475557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766435.243676:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766435.499584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766436.267568:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766436.523567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766436.715585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766436.715595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766436.715599:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766436.715609:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2165 00000100:00080000:1.0:1775766436.715650:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766436.715656:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766436.715688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861402752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766436.715710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766436.715713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766436.715715:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766436.715717:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2165 00000100:00080000:1.0:1775766436.715720:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766436.715722:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766436.715731:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766436.715739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861402880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766436.715747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766436.715751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766436.715753:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766436.715754:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861402752:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766436.715759:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2165 00000100:00080000:1.0:1775766436.715764:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766436.715767:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766436.715770:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766436.715777:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861403008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766436.715825:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766436.715903:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766436/real 1775766436] req@ffffa0e9c8dedc00 x1862023861402752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766436.715922:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861402752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766436.715934:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766436.715939:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766436.715943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861402752:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766436.715967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766436.715991:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766436.716004:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766436/real 1775766436] req@ffffa0e9c8def480 x1862023861402880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766436.716040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861402880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766436.716049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766436.716063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766436.716066:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861402880:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766436.716070:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766436.716088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766436.716096:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766436/real 1775766436] req@ffffa0e9c8def800 x1862023861403008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766436.716104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861403008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766436.716110:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766436.716112:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766436.716114:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403008:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766437.291676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766437.547564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766438.315617:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766438.571584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766439.339676:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766439.595560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766440.363608:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766440.619545:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766441.387653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766441.643572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766441.835574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766441.835584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766441.835588:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766441.835597:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2170 00000100:00080000:2.0:1775766441.835624:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766441.835630:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766441.835664:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861403136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766441.835693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766441.835697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766441.835699:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766441.835702:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2170 00000100:00080000:2.0:1775766441.835705:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766441.835709:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766441.835718:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766441.835728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861403264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766441.835736:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861403136:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766441.835739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766441.835744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766441.835745:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766441.835748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2170 00000100:00080000:2.0:1775766441.835751:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766441.835754:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766441.835759:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766441.835773:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861403392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766441.835796:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766441.835837:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766441/real 1775766441] req@ffffa0e9d0713100 x1862023861403136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766441.835851:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861403136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766441.835867:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766441.835874:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766441.835878:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861403136:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766441.835899:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766441.835917:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766441.835926:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766441/real 1775766441] req@ffffa0e9d0712300 x1862023861403264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766441.835933:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861403264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766441.835939:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766441.835941:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766441.835943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766441.835946:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766441.835956:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766441.835961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766441/real 1775766441] req@ffffa0e9d0711880 x1862023861403392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766441.836001:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861403392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766441.836010:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766441.836013:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766441.836015:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403392:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766442.411598:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766442.667555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766443.435707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766443.691566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766444.459720:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766444.715581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766445.483561:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766445.739561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766446.507708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766446.763567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766446.955591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766446.955599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766446.955603:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766446.955611:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2175 00000100:00080000:2.0:1775766446.955621:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766446.955626:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766446.955662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861403520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766446.955686:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766446.955689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766446.955690:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766446.955693:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2175 00000100:00080000:2.0:1775766446.955695:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766446.955697:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766446.955705:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766446.955712:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861403648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766446.955719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766446.955721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766446.955723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766446.955726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2175 00000100:00080000:2.0:1775766446.955746:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766446.955750:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766446.955755:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766446.955762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861403776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766446.955792:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861403520:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766446.955884:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766446.955954:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766446/real 1775766446] req@ffffa0e9d0711500 x1862023861403520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766446.955977:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861403520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766446.955991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766446.955996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766446.956001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861403520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766446.956010:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766446.956038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766446.956052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766446/real 1775766446] req@ffffa0e9d0713480 x1862023861403648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766446.956065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861403648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766446.956074:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766446.956078:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766446.956082:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403648:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766446.956115:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766446.956139:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766446.956166:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766446/real 1775766446] req@ffffa0e9d0712a00 x1862023861403776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766446.956180:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861403776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766446.956189:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766446.956193:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766446.956197:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861403776:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766447.531646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766447.787576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766448.555622:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766448.811588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766449.579658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766449.835557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766450.603724:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766450.859603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766451.627661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766451.883570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766452.075617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766452.075625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766452.075629:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766452.075637:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2180 00000100:00080000:1.0:1775766452.075646:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766452.075651:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766452.075686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861403904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766452.075709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766452.075712:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766452.075713:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766452.075716:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2180 00000100:00080000:1.0:1775766452.075718:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766452.075721:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766452.075728:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766452.075737:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861404032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766452.075745:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766452.075747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766452.075749:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766452.075752:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861403904:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766452.075754:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2180 00000100:00080000:1.0:1775766452.075774:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766452.075777:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766452.075782:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766452.075791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861404160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766452.075812:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766452.075879:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766452/real 1775766452] req@ffffa0e9c8deed80 x1862023861403904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.075894:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861403904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.075904:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.075908:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766452.075912:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861403904:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766452.075935:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766452.075958:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766452.075970:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766452/real 1775766452] req@ffffa0e9c8dec380 x1862023861404032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.075991:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861404032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.075997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.075999:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766452.076002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766452.076006:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766452.076020:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766452.076027:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766452/real 1775766452] req@ffffa0e9c8ded500 x1862023861404160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.076035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861404160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.076041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.076043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766452.076045:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404160:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766452.651730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766452.907602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766453.675659:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766453.931561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766454.699672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766454.955618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766455.723661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766455.979586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766456.747771:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766457.003577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766457.195594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766457.195603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766457.195607:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766457.195615:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2185 00000100:00080000:1.0:1775766457.195626:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766457.195631:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766457.195682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861404288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766457.195710:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766457.195713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766457.195714:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766457.195717:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2185 00000100:00080000:1.0:1775766457.195720:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766457.195722:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766457.195736:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766457.195750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861404416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766457.195783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766457.195787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766457.195788:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766457.195791:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2185 00000100:00080000:1.0:1775766457.195805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766457.195823:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766457.195827:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766457.195842:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861404544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766457.195853:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861404288:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766457.195979:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766457.196047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766457/real 1775766457] req@ffffa0e9c8dee300 x1862023861404288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766457.196072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861404288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766457.196103:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766457.196108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766457.196113:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861404288:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766457.196152:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766457.196196:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766457.196220:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766457/real 1775766457] req@ffffa0e9c8dec000 x1862023861404416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766457.196261:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861404416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766457.196277:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766457.196284:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766457.196288:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404416:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766457.196297:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766457.196327:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766457.196340:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766457/real 1775766457] req@ffffa0e9c8ded180 x1862023861404544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766457.196354:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861404544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766457.196363:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766457.196367:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766457.196370:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404544:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766457.771596:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766458.027562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766458.795631:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766459.051572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766459.819679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766460.075573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766460.843626:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766461.099634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766461.867639:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766462.123544:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766462.315556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766462.315563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766462.315567:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766462.315575:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2190 00000100:00080000:1.0:1775766462.315584:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766462.315589:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766462.315626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861404672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766462.315648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766462.315650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766462.315652:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766462.315654:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2190 00000100:00080000:1.0:1775766462.315679:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766462.315683:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766462.315692:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766462.315704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861404800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766462.315705:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861404672:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766462.315719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766462.315732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766462.315734:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766462.315737:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2190 00000100:00080000:1.0:1775766462.315740:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766462.315742:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766462.315745:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766462.315764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861404928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766462.315804:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766462.315847:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766462/real 1775766462] req@ffffa0e9c8ded180 x1862023861404672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766462.315863:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861404672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766462.315872:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766462.315875:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766462.315878:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861404672:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766462.315896:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766462.315924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766462.315938:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766462/real 1775766462] req@ffffa0e9c8dec000 x1862023861404800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766462.315952:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861404800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766462.315961:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766462.315965:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766462.315968:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404800:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766462.315972:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766462.315988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766462.316002:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766462/real 1775766462] req@ffffa0e9c8dee300 x1862023861404928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766462.316011:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861404928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766462.316016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766462.316018:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766462.316021:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861404928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766462.891603:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766463.147562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766463.915733:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766464.171632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766464.939615:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766465.195565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766465.963755:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766466.219610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766466.987653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766467.243594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766467.435628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766467.435638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766467.435644:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766467.435656:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2195 00000100:00080000:2.0:1775766467.435691:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766467.435698:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766467.435769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861405056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766467.435794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766467.435797:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766467.435799:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766467.435802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2195 00000100:00080000:2.0:1775766467.435805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766467.435808:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766467.435818:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766467.435826:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861405184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766467.435835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766467.435838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766467.435840:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766467.435842:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861405056:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766467.435845:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2195 00000100:00080000:2.0:1775766467.435851:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766467.435853:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766467.435856:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766467.435864:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861405312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766467.435920:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766467.435961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766467/real 1775766467] req@ffffa0e9d0710e00 x1862023861405056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766467.435977:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861405056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766467.435985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766467.435988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766467.435992:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861405056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766467.436040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766467.436069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766467.436085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766467/real 1775766467] req@ffffa0e9d0710a80 x1862023861405184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766467.436113:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861405184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766467.436120:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766467.436123:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766467.436125:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405184:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766467.436130:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766467.436151:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766467.436163:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766467/real 1775766467] req@ffffa0e9d0712680 x1862023861405312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766467.436191:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861405312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766467.436199:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766467.436203:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766467.436206:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405312:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766468.011687:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766468.267626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766469.035778:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766469.291603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766470.059746:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766470.315582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766471.083703:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766471.339656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766472.107639:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766472.363628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766472.555704:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766472.555715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766472.555721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766472.555731:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2201 00000100:00080000:2.0:1775766472.555744:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766472.555751:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766472.555794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861405440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766472.555827:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766472.555830:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766472.555832:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766472.555835:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2201 00000100:00080000:2.0:1775766472.555837:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766472.555839:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766472.555849:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766472.555857:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861405568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766472.555865:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766472.555868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766472.555869:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766472.555872:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2201 00000100:00080000:2.0:1775766472.555892:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766472.555896:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766472.555901:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766472.555904:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861405440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766472.555912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861405696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766472.555983:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766472.556039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766472/real 1775766472] req@ffffa0e9d0712d80 x1862023861405440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766472.556056:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861405440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766472.556073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766472.556077:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766472.556080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861405440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766472.556086:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766472.556115:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766472.556134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766472/real 1775766472] req@ffffa0e9d0710700 x1862023861405568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766472.556142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861405568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766472.556147:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766472.556149:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766472.556151:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405568:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766472.556187:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766472.556199:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766472.556205:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766472/real 1775766472] req@ffffa0e9d0711f80 x1862023861405696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766472.556212:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861405696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766472.556218:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766472.556219:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766472.556221:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405696:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766473.131680:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766473.387656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766474.155761:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766474.411641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766475.179660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766475.435586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766476.203603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766476.459597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766477.227655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766477.483573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766477.675658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766477.675671:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766477.675676:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766477.675687:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2206 00000100:00080000:1.0:1775766477.675700:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766477.675708:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766477.675773:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861405824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766477.675811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766477.675816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766477.675819:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766477.675825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2206 00000100:00080000:1.0:1775766477.675832:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766477.675838:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766477.675850:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766477.675883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861405952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766477.675892:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861405824:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766477.675906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766477.675912:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766477.675915:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766477.675922:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2206 00000100:00080000:1.0:1775766477.675943:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766477.675949:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766477.675955:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766477.675969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861406080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766477.675980:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766477.676023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766477/real 1775766477] req@ffffa0e9c8ded500 x1862023861405824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766477.676040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861405824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766477.676049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766477.676053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766477.676056:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861405824:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766477.676076:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766477.676093:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766477.676102:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766477/real 1775766477] req@ffffa0e9c8dec380 x1862023861405952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766477.676112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861405952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766477.676118:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766477.676121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766477.676124:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861405952:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766477.676128:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766477.676139:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766477.676156:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766477/real 1775766477] req@ffffa0e9c8deed80 x1862023861406080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766477.676166:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861406080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766477.676172:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766477.676174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766477.676177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766478.251709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766478.507608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766479.275700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766479.531625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766480.299671:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766480.555631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766481.323729:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766481.579609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766482.347702:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766482.603594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766482.795687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766482.795702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766482.795737:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766482.795753:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2211 00000100:00080000:1.0:1775766482.795770:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766482.795781:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766482.795836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861406208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766482.795904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766482.795909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766482.795911:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766482.795916:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2211 00000100:00080000:1.0:1775766482.795921:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766482.795926:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766482.795942:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766482.795968:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861406336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766482.795993:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861406208:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766482.795995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766482.796030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766482.796036:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766482.796046:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2211 00000100:00080000:1.0:1775766482.796057:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766482.796068:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766482.796077:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766482.796097:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861406464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766482.796115:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766482.796170:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766482/real 1775766482] req@ffffa0e9c8def800 x1862023861406208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766482.796191:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861406208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766482.796201:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766482.796205:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766482.796208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861406208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766482.796232:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766482.796251:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766482.796279:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766482/real 1775766482] req@ffffa0e9c8def480 x1862023861406336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766482.796316:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861406336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766482.796324:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766482.796327:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766482.796330:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766482.796335:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766482.796354:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766482.796362:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766482/real 1775766482] req@ffffa0e9c8dedc00 x1862023861406464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766482.796372:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861406464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766482.796379:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766482.796382:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766482.796384:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406464:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766483.371776:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766483.627786:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766484.395823:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766484.651705:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766485.419679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766485.675572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766486.443603:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766486.699643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766487.467665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766487.723632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766487.915715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766487.915725:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766487.915730:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766487.915741:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2216 00000100:00080000:2.0:1775766487.915753:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766487.915760:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766487.915807:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861406592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766487.915845:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766487.915870:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766487.915873:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766487.915880:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2216 00000100:00080000:2.0:1775766487.915887:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766487.915895:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766487.915909:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766487.915920:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861406592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766487.915933:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861406720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766487.915952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766487.915956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766487.915958:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766487.915964:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2216 00000100:00080000:2.0:1775766487.915987:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766487.915991:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766487.915997:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766487.916012:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775766487.916017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861406848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766487.916095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766487/real 1775766487] req@ffffa0e9d0711c00 x1862023861406592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766487.916114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861406592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766487.916136:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766487.916141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766487.916145:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861406592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766487.916173:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766487.916207:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766487.916226:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766487/real 1775766487] req@ffffa0e9d0713b80 x1862023861406720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766487.916245:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861406720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766487.916255:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766487.916259:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766487.916262:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406720:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766487.916268:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766487.916288:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766487.916298:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766487/real 1775766487] req@ffffa0e9d0710000 x1862023861406848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766487.916308:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861406848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766487.916316:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766487.916319:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766487.916322:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861406848:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766488.491669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766488.747589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766489.515729:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766489.771613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766490.539729:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766490.795589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766491.563656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766491.819572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766492.587696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766492.843580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766493.035640:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766493.035653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766493.035662:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766493.035679:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2221 00000100:00080000:1.0:1775766493.035697:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766493.035709:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766493.035770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861406976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766493.035812:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766493.035816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766493.035818:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766493.035823:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2221 00000100:00080000:1.0:1775766493.035828:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766493.035832:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766493.035845:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766493.035861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861407104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766493.035872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766493.035876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766493.035902:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766493.035910:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2221 00000100:00080000:1.0:1775766493.035917:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766493.035925:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861406976:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766493.035930:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766493.035944:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766493.035968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861407232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766493.036070:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407104:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766493.036113:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407232:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766493.611742:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766493.867656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766494.635642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766494.891578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766495.659722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766495.915592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775766496.107882:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775766496.107938:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766496.107959:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766496.108021:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766496.108034:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766496.108172:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766493/real 1775766496] req@ffffa0e9c8deea00 x1862023861407232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766496.108222:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861407232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766496.108243:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766496.108252:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766496.108260:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766496.108276:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766493/real 1775766496] req@ffffa0e9c8dee680 x1862023861406976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766496.108300:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861406976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766496.108314:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766496.108322:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766496.108329:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861406976:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766496.108343:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766493/real 1775766496] req@ffffa0e9c8ded880 x1862023861407104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766496.108432:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861407104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766496.108448:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766496.108455:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766496.108462:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407104:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766496.683610:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766496.939582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766497.707626:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766497.963634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766498.155667:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766498.155677:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766498.155683:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766498.155694:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2226 00000100:00080000:3.0:1775766498.155710:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766498.155717:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766498.155760:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861407360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766498.155804:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766498.155809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766498.155811:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766498.155815:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2226 00000100:00080000:3.0:1775766498.155821:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766498.155825:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766498.155837:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766498.155854:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861407488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766498.155867:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766498.155872:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766498.155874:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766498.155879:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2226 00000100:00080000:3.0:1775766498.155884:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766498.155890:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766498.155901:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861407360:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775766498.155921:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766498.155941:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861407616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766498.156006:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766498.156095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766498/real 1775766498] req@ffffa0e9d0543800 x1862023861407360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766498.156120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861407360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766498.156133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766498.156139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766498.156143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861407360:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766498.156154:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766498.156179:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766498.156195:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766498/real 1775766498] req@ffffa0e9d0540700 x1862023861407488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766498.156208:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861407488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766498.156238:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766498.156242:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766498.156245:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407488:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766498.156280:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766498.156305:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766498.156316:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766498/real 1775766498] req@ffffa0e9d0540e00 x1862023861407616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766498.156327:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861407616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766498.156334:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766498.156338:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766498.156341:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407616:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766498.731704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766498.987567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766499.755667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766500.011611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766500.779613:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766501.035548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766501.803701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766502.059599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766502.827699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766503.083607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766503.275620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766503.275630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766503.275634:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766503.275642:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2231 00000100:00080000:1.0:1775766503.275651:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766503.275656:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766503.275692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861407744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766503.275716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766503.275719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766503.275720:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766503.275723:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2231 00000100:00080000:1.0:1775766503.275725:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766503.275727:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766503.275735:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766503.275757:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861407744:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766503.275766:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861407872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766503.275775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766503.275778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766503.275779:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766503.275781:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2231 00000100:00080000:1.0:1775766503.275785:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766503.275787:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766503.275791:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766503.275798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861408000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766503.275820:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766503.275861:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766503/real 1775766503] req@ffffa0e9c8deca80 x1862023861407744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766503.275876:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861407744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766503.275891:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766503.275901:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766503.275904:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861407744:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766503.275922:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766503.275940:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766503.275949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766503/real 1775766503] req@ffffa0e9c8dedf80 x1862023861407872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766503.275957:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861407872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766503.275963:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766503.275965:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766503.275967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861407872:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766503.275971:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766503.275980:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766503.275986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766503/real 1775766503] req@ffffa0e9c8dece00 x1862023861408000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766503.275995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861408000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766503.276003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766503.276006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766503.276009:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408000:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766503.851607:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766504.107585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766504.875706:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766505.131609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766505.899719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766506.155571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766506.923625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766507.179576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766507.947672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766508.203592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766508.395580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766508.395589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766508.395593:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766508.395601:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2236 00000100:00080000:2.0:1775766508.395610:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766508.395649:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766508.395683:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861408128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766508.395706:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766508.395708:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766508.395710:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766508.395712:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2236 00000100:00080000:2.0:1775766508.395715:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766508.395718:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766508.395725:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766508.395746:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861408128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766508.395754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861408256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766508.395762:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766508.395764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766508.395765:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766508.395768:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2236 00000100:00080000:2.0:1775766508.395771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766508.395773:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766508.395777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766508.395786:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861408384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766508.395812:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766508.395854:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766508/real 1775766508] req@ffffa0e9d0711180 x1862023861408128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766508.395870:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861408128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766508.395879:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766508.395882:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766508.395885:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861408128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766508.395905:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766508.395920:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766508.395928:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766508/real 1775766508] req@ffffa0e9d0710380 x1862023861408256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766508.395935:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861408256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766508.395941:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766508.395943:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766508.395945:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766508.395949:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766508.395958:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766508.395963:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766508/real 1775766508] req@ffffa0e9d0710000 x1862023861408384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766508.395982:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861408384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766508.395987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766508.395989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766508.395991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408384:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766508.971657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766509.227565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766509.995635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766510.251576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766511.019669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766511.275635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766512.043629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766512.299599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766513.067619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766513.323631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766513.515606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766513.515616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766513.515622:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766513.515633:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2242 00000100:00080000:0.0:1775766513.515645:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766513.515651:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766513.515701:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861408512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766513.515748:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766513.515751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766513.515753:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766513.515756:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2242 00000100:00080000:0.0:1775766513.515760:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766513.515763:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766513.515773:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766513.515800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861408640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766513.515808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766513.515810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766513.515812:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766513.515815:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2242 00000100:00080000:0.0:1775766513.515818:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766513.515821:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766513.515824:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766513.515831:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861408768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766513.515841:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861408512:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766513.515919:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766513.515959:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766513/real 1775766513] req@ffffa0e9d09de680 x1862023861408512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766513.515975:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861408512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766513.515983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766513.515987:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766513.515990:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861408512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766513.516005:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766513.516022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766513.516031:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766513/real 1775766513] req@ffffa0e9d09dca80 x1862023861408640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766513.516039:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861408640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766513.516045:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766513.516047:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766513.516049:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408640:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766513.516070:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766513.516080:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766513.516086:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766513/real 1775766513] req@ffffa0e9d09ddf80 x1862023861408768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766513.516094:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861408768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766513.516099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766513.516101:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766513.516103:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861408768:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766514.091688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766514.347587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766515.115692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766515.371602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766516.139683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766516.395619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766517.163673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766517.419603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766518.187668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766518.443576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766518.635632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766518.635649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766518.635654:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766518.635667:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2247 00000100:00080000:0.0:1775766518.635682:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766518.635690:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766518.635738:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861408896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766518.635773:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766518.635777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766518.635779:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766518.635801:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2247 00000100:00080000:0.0:1775766518.635806:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766518.635810:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766518.635821:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766518.635834:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861409024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766518.635837:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861408896:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766518.635847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766518.635850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766518.635851:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766518.635857:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2247 00000100:00080000:0.0:1775766518.635861:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766518.635864:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766518.635867:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766518.635875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861409152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766518.635929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766518.635974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766518/real 1775766518] req@ffffa0e9d09ddc00 x1862023861408896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766518.635991:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861408896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766518.636000:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766518.636003:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766518.636007:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861408896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766518.636028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766518.636044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766518.636052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766518/real 1775766518] req@ffffa0e9d09dd180 x1862023861409024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766518.636060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861409024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766518.636066:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766518.636067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766518.636069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766518.636073:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766518.636083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766518.636091:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766518/real 1775766518] req@ffffa0e9d09dd500 x1862023861409152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766518.636118:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861409152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766518.636126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766518.636130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766518.636134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409152:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766519.211610:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766519.467598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766520.235663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766520.491564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766521.259639:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766521.515561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766522.283657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766522.539541:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766523.307675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766523.563630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766523.755643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766523.755653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766523.755657:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766523.755666:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2252 00000100:00080000:3.0:1775766523.755677:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766523.755685:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766523.755729:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861409280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766523.755761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766523.755765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766523.755767:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766523.755771:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2252 00000100:00080000:3.0:1775766523.755775:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766523.755780:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766523.755793:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766523.755831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861409408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766523.755834:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861409280:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766523.755855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766523.755858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766523.755859:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766523.755863:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2252 00000100:00080000:3.0:1775766523.755867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766523.755870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766523.755875:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766523.755884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861409536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766523.755941:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766523.755997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766523/real 1775766523] req@ffffa0e9d0540a80 x1862023861409280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766523.756019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861409280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766523.756032:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766523.756037:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766523.756041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861409280:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766523.756074:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766523.756101:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766523.756131:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766523/real 1775766523] req@ffffa0e9d0542d80 x1862023861409408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766523.756146:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861409408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766523.756168:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766523.756172:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766523.756176:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409408:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766523.756183:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766523.756204:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766523.756212:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766523/real 1775766523] req@ffffa0e9d0541f80 x1862023861409536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766523.756220:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861409536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766523.756226:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766523.756228:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766523.756230:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409536:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766524.331712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766524.587607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766525.355621:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766525.611579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766526.379706:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766526.635635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766527.403639:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766527.659600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766528.427618:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766528.683624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766528.875595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766528.875603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766528.875609:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766528.875619:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2257 00000100:00080000:1.0:1775766528.875632:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766528.875638:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766528.875680:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861409664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766528.875713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766528.875733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766528.875736:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766528.875741:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2257 00000100:00080000:1.0:1775766528.875746:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766528.875752:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766528.875758:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861409664:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766528.875765:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766528.875778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861409792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766528.875787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766528.875790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766528.875792:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766528.875795:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2257 00000100:00080000:1.0:1775766528.875798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766528.875801:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766528.875804:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766528.875811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861409920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766528.875831:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766528.875878:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766528/real 1775766528] req@ffffa0e9c8dec700 x1862023861409664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766528.875896:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861409664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766528.875912:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766528.875915:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766528.875920:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861409664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766528.875941:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766528.875960:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766528.875968:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766528/real 1775766528] req@ffffa0e9c8defb80 x1862023861409792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766528.875976:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861409792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766528.875982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766528.875984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766528.875986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409792:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766528.875989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766528.875999:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766528.876005:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766528/real 1775766528] req@ffffa0e9c8def100 x1862023861409920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766528.876012:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861409920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766528.876027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766528.876029:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766528.876030:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861409920:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766529.451614:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766529.707568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766530.475715:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766530.731597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766531.499632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766531.755581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766532.523714:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766532.779567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766533.547654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766533.803591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766533.995547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766533.995555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766533.995561:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766533.995571:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2262 00000100:00080000:1.0:1775766533.995582:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766533.995606:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766533.995647:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861410048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766533.995678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766533.995681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766533.995682:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766533.995685:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2262 00000100:00080000:1.0:1775766533.995689:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766533.995692:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766533.995702:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766533.995711:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861410176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766533.995718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766533.995720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766533.995721:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766533.995724:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2262 00000100:00080000:1.0:1775766533.995726:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766533.995728:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766533.995730:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766533.995736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861410304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766533.995768:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861410048:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766533.995838:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766533.995885:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766533/real 1775766533] req@ffffa0e9c8def100 x1862023861410048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766533.995903:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861410048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766533.995911:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766533.995915:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766533.995918:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861410048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766533.995923:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766533.995938:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766533.995949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766533/real 1775766533] req@ffffa0e9c8defb80 x1862023861410176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766533.995962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861410176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766533.995971:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766533.995975:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766533.995978:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766533.995985:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766533.996030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766533.996039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766533/real 1775766533] req@ffffa0e9c8dec700 x1862023861410304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766533.996048:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861410304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766533.996054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766533.996056:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766533.996058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410304:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766534.571678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766534.827627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766535.595674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766535.851610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766536.619727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766536.875619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766537.643649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766537.899551:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766538.667688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766538.923578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766539.115575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766539.115595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766539.115601:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766539.115612:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2267 00000100:00080000:1.0:1775766539.115626:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766539.115635:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766539.115682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861410432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766539.115715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766539.115719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766539.115721:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766539.115725:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2267 00000100:00080000:1.0:1775766539.115729:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766539.115733:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766539.115751:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766539.115765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861410560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766539.115777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766539.115781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766539.115785:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766539.115786:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861410432:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766539.115791:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2267 00000100:00080000:1.0:1775766539.115799:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766539.115803:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766539.115808:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766539.115837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861410688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766539.115877:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766539.115937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766539/real 1775766539] req@ffffa0e9c8dece00 x1862023861410432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766539.115978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861410432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766539.116009:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766539.116014:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766539.116018:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861410432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766539.116045:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766539.116069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766539.116083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766539/real 1775766539] req@ffffa0e9c8dedf80 x1862023861410560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766539.116109:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861410560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766539.116126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766539.116130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766539.116133:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410560:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766539.116140:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766539.116162:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766539.116173:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766539/real 1775766539] req@ffffa0e9c8deca80 x1862023861410688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766539.116185:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861410688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766539.116193:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766539.116197:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766539.116200:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410688:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766539.691696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766539.947603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766540.715608:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766540.971724:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766541.739772:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766541.995649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766542.763647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766543.019611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766543.787643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766544.043657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766544.235632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766544.235646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766544.235654:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766544.235667:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2272 00000100:00080000:1.0:1775766544.235684:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766544.235694:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766544.235751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861410816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766544.235799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766544.235804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766544.235807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766544.235812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2272 00000100:00080000:1.0:1775766544.235816:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766544.235820:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766544.235834:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766544.235848:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861410944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766544.235862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766544.235868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766544.235871:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766544.235875:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2272 00000100:00080000:1.0:1775766544.235930:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766544.235944:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766544.235963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861410816:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766544.235970:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766544.235992:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861411072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766544.236072:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766544.236143:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766544/real 1775766544] req@ffffa0e9c8ded880 x1862023861410816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766544.236173:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861410816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766544.236194:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766544.236199:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766544.236204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861410816:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766544.236212:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766544.236240:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766544.236254:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766544/real 1775766544] req@ffffa0e9c8dee680 x1862023861410944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766544.236266:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861410944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766544.236281:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766544.236284:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766544.236287:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861410944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766544.236314:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766544.236329:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766544.236338:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766544/real 1775766544] req@ffffa0e9c8deea00 x1862023861411072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766544.236349:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861411072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766544.236356:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766544.236359:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766544.236361:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411072:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766544.811656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766545.067555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766545.835791:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766546.091671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766546.859682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766547.115601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766547.883742:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766548.139615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766548.907694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766549.163616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766549.355725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766549.355755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766549.355763:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766549.355777:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2277 00000100:00080000:3.0:1775766549.355796:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766549.355807:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766549.355895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861411200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766549.355979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766549.355987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766549.355991:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766549.356000:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2277 00000100:00080000:3.0:1775766549.356008:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766549.356016:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766549.356031:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766549.356051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861411328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766549.356065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766549.356070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766549.356073:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766549.356078:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2277 00000100:00080000:3.0:1775766549.356082:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766549.356086:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766549.356091:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766549.356101:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861411456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766549.356184:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861411200:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766549.356316:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766549.356380:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766549/real 1775766549] req@ffffa0e9d0541180 x1862023861411200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766549.356407:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861411200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766549.356422:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766549.356429:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766549.356434:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861411200:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766549.356444:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766549.356471:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766549.356540:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766549/real 1775766549] req@ffffa0e9d0543480 x1862023861411328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766549.356558:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861411328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766549.356570:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766549.356574:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766549.356578:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411328:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766549.356586:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766549.356636:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766549.356658:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766549/real 1775766549] req@ffffa0e9d0543100 x1862023861411456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766549.356683:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861411456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766549.356700:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766549.356707:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766549.356714:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411456:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766549.931666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766550.187618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766550.955703:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766551.211640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766551.979901:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766552.235595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766553.003626:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766553.259576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766554.027813:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766554.283646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766554.475655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766554.475666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766554.475671:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766554.475682:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2282 00000100:00080000:1.0:1775766554.475696:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766554.475704:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766554.475786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861411584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766554.475857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766554.475860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766554.475862:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766554.475865:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2282 00000100:00080000:1.0:1775766554.475868:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766554.475872:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766554.475883:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766554.475895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861411712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766554.475910:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861411584:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766554.475926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766554.475929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766554.475931:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766554.475936:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2282 00000100:00080000:1.0:1775766554.475939:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766554.475942:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766554.475947:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766554.475959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861411840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766554.475997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766554.476037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766554/real 1775766554] req@ffffa0e9c8dedc00 x1862023861411584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766554.476055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861411584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766554.476064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766554.476068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766554.476071:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861411584:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766554.476092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766554.476109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766554.476118:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766554/real 1775766554] req@ffffa0e9c8def480 x1862023861411712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766554.476139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861411712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766554.476146:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766554.476148:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766554.476151:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411712:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766554.476155:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766554.476169:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766554.476175:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766554/real 1775766554] req@ffffa0e9c8def800 x1862023861411840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766554.476184:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861411840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766554.476190:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766554.476191:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766554.476193:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861411840:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766555.051775:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766555.307597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766556.075689:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766556.331567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766557.099698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766557.355578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766558.123621:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766558.379609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766559.147635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766559.403552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766559.595723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766559.595773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766559.595783:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766559.595803:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2288 00000100:00080000:1.0:1775766559.595825:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766559.595840:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766559.595910:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861411968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766559.595966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766559.595971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766559.595974:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766559.595981:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2288 00000100:00080000:1.0:1775766559.596000:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766559.596006:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766559.596041:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861411968:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766559.596049:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766559.596084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861412096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766559.596102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766559.596107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766559.596109:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766559.596115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2288 00000100:00080000:1.0:1775766559.596133:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766559.596138:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766559.596146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766559.596158:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861412224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766559.596243:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412096:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766559.596286:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412224:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766560.171764:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766560.427584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766561.195823:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766561.451701:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766562.219626:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766562.475575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766562.667864:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766562.667912:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766562.667934:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766562.667989:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766562.668000:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766562.668054:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766559/real 1775766562] req@ffffa0e9c8dec380 x1862023861412096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766562.668090:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861412096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766562.668105:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766562.668112:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766562.668118:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412096:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766562.668128:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766559/real 1775766562] req@ffffa0e9c8ded500 x1862023861412224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766562.668143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861412224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766562.668152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766562.668156:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766562.668160:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766562.668169:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766559/real 1775766562] req@ffffa0e9c8deed80 x1862023861411968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766562.668192:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861411968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766562.668202:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766562.668206:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766562.668210:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861411968:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775766563.243735:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766563.499636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766564.267734:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766564.523565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766564.715629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766564.715641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766564.715647:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766564.715660:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2293 00000100:00080000:3.0:1775766564.715698:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766564.715708:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766564.715758:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861412352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766564.715792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766564.715797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766564.715799:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766564.715804:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2293 00000100:00080000:3.0:1775766564.715808:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766564.715813:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766564.715827:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766564.715845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861412480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766564.715859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861412352:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766564.715861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766564.715884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766564.715886:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766564.715893:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2293 00000100:00080000:3.0:1775766564.715899:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766564.715904:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766564.715911:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766564.715943:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861412608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766564.715970:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766564.716025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766564/real 1775766564] req@ffffa0e9d0543100 x1862023861412352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766564.716055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861412352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766564.716069:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766564.716074:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766564.716079:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861412352:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766564.716113:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766564.716143:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766564.716177:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766564/real 1775766564] req@ffffa0e9d0543480 x1862023861412480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766564.716195:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861412480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766564.716206:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766564.716210:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766564.716214:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412480:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766564.716221:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766564.716247:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766564.716260:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766564/real 1775766564] req@ffffa0e9d0541180 x1862023861412608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766564.716293:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861412608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766564.716303:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766564.716306:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766564.716310:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412608:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766565.291642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766565.547545:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766566.315826:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766566.571681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766567.339873:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766567.595718:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766568.363718:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766568.619608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766569.387699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766569.643716:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766569.835575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766569.835583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766569.835588:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766569.835598:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2298 00000100:00080000:3.0:1775766569.835609:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766569.835616:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766569.835675:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861412736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766569.835701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766569.835704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766569.835706:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766569.835709:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2298 00000100:00080000:3.0:1775766569.835712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766569.835714:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766569.835724:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766569.835732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861412864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766569.835741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766569.835745:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861412736:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766569.835745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766569.835748:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766569.835767:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2298 00000100:00080000:3.0:1775766569.835771:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766569.835774:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766569.835778:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766569.835788:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861412992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766569.835817:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766569.835864:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766569/real 1775766569] req@ffffa0e9d0541f80 x1862023861412736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766569.835881:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861412736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766569.835890:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766569.835894:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766569.835897:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861412736:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766569.835920:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766569.835936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766569.835945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766569/real 1775766569] req@ffffa0e9d0542d80 x1862023861412864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766569.835961:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861412864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766569.835978:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766569.835981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766569.835983:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412864:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766569.835988:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766569.836003:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766569.836010:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766569/real 1775766569] req@ffffa0e9d0540a80 x1862023861412992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766569.836018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861412992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766569.836023:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766569.836026:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766569.836028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861412992:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775766570.411656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766570.667687:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766571.435656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766571.691693:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766572.459668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766572.715749:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766573.483826:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766573.739652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766574.507725:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766574.763553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766574.955661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766574.955670:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766574.955676:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766574.955686:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2303 00000100:00080000:3.0:1775766574.955699:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766574.955705:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766574.955749:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861413120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766574.955776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766574.955779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766574.955781:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766574.955784:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2303 00000100:00080000:3.0:1775766574.955786:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766574.955788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766574.955798:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766574.955816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861413248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766574.955839:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861413120:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766574.955847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766574.955850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766574.955852:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766574.955856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2303 00000100:00080000:3.0:1775766574.955859:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766574.955861:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766574.955866:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766574.955876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861413376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766574.955916:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766574.955956:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766574/real 1775766574] req@ffffa0e9d0540e00 x1862023861413120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766574.955972:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861413120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766574.955981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766574.955991:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766574.955994:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861413120:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766574.956017:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766574.956035:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766574.956043:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766574/real 1775766574] req@ffffa0e9d0540700 x1862023861413248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766574.956052:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861413248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766574.956057:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766574.956059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766574.956061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413248:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766574.956065:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766574.956075:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766574.956081:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766574/real 1775766574] req@ffffa0e9d0543800 x1862023861413376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766574.956088:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861413376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766574.956093:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766574.956095:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766574.956097:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413376:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766575.531704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766575.787629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775766576.555712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766576.811604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766577.579670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766577.835615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766578.603898:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766578.859659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766579.627663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766579.883630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766580.075629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766580.075638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766580.075651:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766580.075664:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2308 00000100:00080000:2.0:1775766580.075678:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766580.075685:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766580.075729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861413504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766580.075754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766580.075758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766580.075759:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766580.075763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2308 00000100:00080000:2.0:1775766580.075766:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766580.075770:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766580.075779:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766580.075789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861413504:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766580.075794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861413632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766580.075805:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766580.075808:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766580.075823:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766580.075827:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2308 00000100:00080000:2.0:1775766580.075831:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766580.075834:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766580.075839:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766580.075877:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861413760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766580.075891:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766580.075983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766580/real 1775766580] req@ffffa0e9d0713b80 x1862023861413504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766580.076008:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861413504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766580.076022:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766580.076027:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766580.076043:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861413504:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766580.076068:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766580.076091:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766580.076106:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766580/real 1775766580] req@ffffa0e9d0711c00 x1862023861413632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766580.076131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861413632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766580.076142:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766580.076146:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766580.076150:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413632:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766580.076157:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766580.076175:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766580.076184:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766580/real 1775766580] req@ffffa0e9d0711f80 x1862023861413760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766580.076221:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861413760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766580.076230:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766580.076234:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766580.076238:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861413760:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766580.651748:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766580.907741:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766581.675794:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766581.931625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766582.699734:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766582.955719:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766583.723699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766583.979689:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766584.747703:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766585.003561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766585.195596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766585.195605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766585.195611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766585.195622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2313 00000100:00080000:0.0:1775766585.195633:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766585.195642:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766585.195692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861413888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766585.195735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766585.195738:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766585.195739:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766585.195743:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2313 00000100:00080000:0.0:1775766585.195746:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766585.195750:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766585.195760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766585.195783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861414016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766585.195789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861413888:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766585.195819:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766585.195824:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766585.195826:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766585.195832:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2313 00000100:00080000:0.0:1775766585.195851:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766585.195856:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766585.195862:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766585.195863:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766585.195894:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861414144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766585.195920:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766585/real 1775766585] req@ffffa0e9d09dd880 x1862023861413888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766585.195958:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861413888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766585.195971:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766585.195976:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766585.195981:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861413888:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766585.196018:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766585.196044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766585.196058:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766585/real 1775766585] req@ffffa0e9d09df800 x1862023861414016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766585.196074:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861414016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766585.196084:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766585.196088:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766585.196092:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414016:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766585.196108:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766585.196126:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766585.196135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766585/real 1775766585] req@ffffa0e9d09dc700 x1862023861414144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766585.196143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861414144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766585.196150:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766585.196152:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766585.196154:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414144:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766585.771724:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766586.027637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766586.795686:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766587.051611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766587.819655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766588.075652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766588.843684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766589.099617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766589.867661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766590.123606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766590.315640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766590.315651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766590.315670:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766590.315683:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2318 00000100:00080000:1.0:1775766590.315698:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766590.315706:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766590.315751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861414272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766590.315788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766590.315792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766590.315794:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766590.315799:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2318 00000100:00080000:1.0:1775766590.315802:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766590.315805:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766590.315818:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766590.315830:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861414400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766590.315842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766590.315846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766590.315848:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766590.315851:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2318 00000100:00080000:1.0:1775766590.315854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766590.315857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766590.315862:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766590.315871:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861414272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766590.315876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861414528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766590.315979:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766590.316035:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766590/real 1775766590] req@ffffa0e9c8dee300 x1862023861414272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766590.316064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861414272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766590.316081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766590.316088:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766590.316095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861414272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766590.316144:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766590.316197:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766590.316235:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766590/real 1775766590] req@ffffa0e9c8dec000 x1862023861414400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766590.316281:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861414400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766590.316300:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766590.316307:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766590.316314:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414400:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766590.316328:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766590.316371:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766590.316395:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766590/real 1775766590] req@ffffa0e9c8ded180 x1862023861414528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766590.316469:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861414528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766590.316525:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766590.316532:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766590.316538:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414528:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766590.891598:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766591.147615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766591.915663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766592.171603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766592.939690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766593.195586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766593.963684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766594.219621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766594.987674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766595.243593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766595.435566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766595.435576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766595.435581:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766595.435591:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2323 00000100:00080000:1.0:1775766595.435603:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766595.435610:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766595.435655:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861414656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766595.435682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766595.435685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766595.435687:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766595.435690:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2323 00000100:00080000:1.0:1775766595.435693:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766595.435695:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766595.435705:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766595.435714:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861414784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766595.435723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766595.435726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775766595.435727:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861414656:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766595.435728:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766595.435736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2323 00000100:00080000:1.0:1775766595.435754:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766595.435758:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766595.435763:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766595.435774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861414912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766595.435804:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766595.435843:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766595/real 1775766595] req@ffffa0e9c8ded180 x1862023861414656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766595.435860:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861414656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766595.435879:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766595.435883:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766595.435886:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861414656:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766595.435906:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766595.435924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766595.435933:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766595/real 1775766595] req@ffffa0e9c8dec000 x1862023861414784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766595.435941:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861414784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766595.435947:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766595.435949:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766595.435952:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414784:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766595.435955:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766595.435965:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766595.435971:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766595/real 1775766595] req@ffffa0e9c8dee300 x1862023861414912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766595.435978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861414912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766595.435983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766595.435985:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766595.435986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861414912:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766596.011578:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766596.267586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766597.035722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766597.291647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766598.059756:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766598.315648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766599.083612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766599.339576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766600.107651:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766600.363618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766600.555732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766600.555750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766600.555759:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766600.555777:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2329 00000100:00080000:3.0:1775766600.555810:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766600.555823:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766600.555893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861415040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766600.555979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766600.555984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766600.555987:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766600.555992:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2329 00000100:00080000:3.0:1775766600.555999:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766600.556005:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766600.556019:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766600.556038:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861415168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766600.556054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766600.556060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766600.556062:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766600.556070:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861415040:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766600.556074:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2329 00000100:00080000:3.0:1775766600.556087:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766600.556092:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766600.556099:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766600.556115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861415296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766600.556183:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766600.556240:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766600/real 1775766600] req@ffffa0e9d0542300 x1862023861415040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766600.556267:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861415040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766600.556281:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766600.556287:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766600.556293:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861415040:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766600.556365:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766600.556400:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766600.556418:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766600/real 1775766600] req@ffffa0e9d0540380 x1862023861415168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766600.556444:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861415168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766600.556461:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766600.556467:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766600.556474:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415168:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766600.556541:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766600.556575:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766600.556592:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766600/real 1775766600] req@ffffa0e9d0541c00 x1862023861415296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766600.556624:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861415296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766600.556635:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766600.556639:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766600.556643:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415296:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766601.131695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766601.387675:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766602.155680:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766602.411604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766603.179728:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766603.435572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766604.203716:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766604.459596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766605.227621:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766605.483657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766605.675629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766605.675642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766605.675656:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766605.675690:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2334 00000100:00080000:3.0:1775766605.675702:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766605.675709:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766605.675762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861415424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766605.675803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766605.675806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766605.675807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766605.675811:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2334 00000100:00080000:3.0:1775766605.675814:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766605.675817:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766605.675826:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766605.675837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861415552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766605.675845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766605.675848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766605.675849:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766605.675852:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2334 00000100:00080000:3.0:1775766605.675854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766605.675869:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766605.675872:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766605.675878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861415680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766605.675976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861415424:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766605.676088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766605.676143:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766605/real 1775766605] req@ffffa0e9d0543b80 x1862023861415424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766605.676187:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861415424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766605.676207:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766605.676215:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766605.676222:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861415424:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766605.676235:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766605.676298:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766605.676323:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766605/real 1775766605] req@ffffa0e9d0540000 x1862023861415552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766605.676355:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861415552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766605.676364:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766605.676368:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766605.676372:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415552:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766605.676379:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766605.676400:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766605.676410:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766605/real 1775766605] req@ffffa0e9d0542a00 x1862023861415680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766605.676422:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861415680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766605.676430:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766605.676433:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766605.676436:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415680:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766606.251720:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766606.507614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766607.275716:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766607.531651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766608.299663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766608.555611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766609.323673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766609.579636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766610.347699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766610.603660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766610.795640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766610.795652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766610.795658:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766610.795668:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2339 00000100:00080000:1.0:1775766610.795682:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766610.795690:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766610.795736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861415808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766610.795774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766610.795779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766610.795781:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766610.795786:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2339 00000100:00080000:1.0:1775766610.795791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766610.795797:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766610.795809:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766610.795831:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861415808:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766610.795852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861415936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766610.795870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766610.795875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766610.795877:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766610.795882:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2339 00000100:00080000:1.0:1775766610.795888:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766610.795893:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775766610.795900:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775766610.795904:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766610.795921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861416064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766610.795958:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766610/real 1775766610] req@ffffa0e9c8deed80 x1862023861415808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766610.796009:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861415808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766610.796018:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766610.796022:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766610.796025:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861415808:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766610.796046:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766610.796066:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766610.796075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766610/real 1775766610] req@ffffa0e9c8ded500 x1862023861415936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766610.796084:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861415936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766610.796090:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766610.796092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766610.796095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861415936:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766610.796098:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766610.796109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766610.796115:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766610/real 1775766610] req@ffffa0e9c8dec380 x1862023861416064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766610.796123:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861416064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766610.796139:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766610.796141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766610.796144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416064:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766611.371616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766611.627628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766612.395676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766612.651569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766613.419762:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766613.675726:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766614.443617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766614.699597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766615.467716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766615.723623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766615.915647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766615.915657:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766615.915661:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766615.915670:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2344 00000100:00080000:1.0:1775766615.915680:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766615.915686:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766615.915724:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861416192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766615.915751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766615.915754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766615.915755:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766615.915778:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2344 00000100:00080000:1.0:1775766615.915782:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766615.915802:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766615.915811:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766615.915822:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861416192:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766615.915823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861416320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766615.915837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766615.915839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766615.915840:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766615.915843:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2344 00000100:00080000:1.0:1775766615.915846:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766615.915848:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766615.915851:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766615.915859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861416448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766615.915887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766615.915924:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766615/real 1775766615] req@ffffa0e9c8def800 x1862023861416192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766615.915938:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861416192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766615.915946:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766615.915950:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766615.915953:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861416192:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766615.915971:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766615.915985:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766615.915993:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766615/real 1775766615] req@ffffa0e9c8def480 x1862023861416320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766615.916001:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861416320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766615.916013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766615.916023:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766615.916026:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416320:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766615.916030:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766615.916045:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766615.916052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766615/real 1775766615] req@ffffa0e9c8dedc00 x1862023861416448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766615.916060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861416448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766615.916065:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766615.916067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766615.916069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416448:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766616.491608:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766616.747572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766617.515622:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766617.771612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766618.539640:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766618.795627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766619.563662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766619.819583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766620.587592:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766620.843654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766621.035613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766621.035623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766621.035628:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766621.035638:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2349 00000100:00080000:3.0:1775766621.035650:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766621.035657:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766621.035700:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861416576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766621.035729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766621.035732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766621.035734:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766621.035737:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2349 00000100:00080000:3.0:1775766621.035739:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766621.035741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766621.035751:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766621.035759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861416704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766621.035766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766621.035783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766621.035785:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766621.035787:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2349 00000100:00080000:3.0:1775766621.035790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766621.035793:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766621.035796:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766621.035803:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861416832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766621.035852:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861416576:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766621.035934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766621.035977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766621/real 1775766621] req@ffffa0e9d0541500 x1862023861416576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766621.036003:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861416576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766621.036022:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766621.036026:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766621.036029:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861416576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766621.036035:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766621.036054:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766621.036063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766621/real 1775766621] req@ffffa0e9d0541880 x1862023861416704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766621.036072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861416704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766621.036077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766621.036080:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766621.036082:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416704:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766621.036085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766621.036111:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766621.036117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766621/real 1775766621] req@ffffa0e9d0542680 x1862023861416832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766621.036125:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861416832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766621.036131:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766621.036133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766621.036135:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861416832:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766621.611673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766621.867602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766622.635727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766622.891658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766623.659678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766623.915610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766624.683658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766624.939598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766625.707672:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766625.963601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766626.155638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766626.155648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766626.155654:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766626.155665:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2354 00000100:00080000:2.0:1775766626.155674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766626.155681:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766626.155722:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861416960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766626.155765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766626.155768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766626.155769:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766626.155772:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2354 00000100:00080000:2.0:1775766626.155793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766626.155796:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766626.155805:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766626.155816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861417088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766626.155824:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766626.155826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766626.155827:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766626.155830:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2354 00000100:00080000:2.0:1775766626.155832:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766626.155835:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766626.155839:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766626.155846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861417216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766626.155897:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861416960:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766626.156130:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766626.156174:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417216:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766626.731627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766626.987592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766627.755733:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766628.011631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766628.779647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766629.035640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775766629.227875:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775766629.227922:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766629.227945:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766629.227995:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766629.228006:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766629.228072:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766626/real 1775766629] req@ffffa0e9d0710700 x1862023861416960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766629.228104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861416960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766629.228118:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766629.228124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766629.228129:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861416960:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766629.228138:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766626/real 1775766629] req@ffffa0e9d0712d80 x1862023861417088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766629.228154:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861417088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766629.228164:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766629.228168:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766629.228172:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766629.228180:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766626/real 1775766629] req@ffffa0e9d0712680 x1862023861417216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766629.228194:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861417216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766629.228225:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766629.228231:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766629.228235:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417216:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766629.803639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766630.059605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766630.827697:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766631.083562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766631.275656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766631.275665:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766631.275670:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766631.275680:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2359 00000100:00080000:3.0:1775766631.275691:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766631.275697:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766631.275735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861417344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766631.275779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766631.275782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766631.275784:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766631.275787:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2359 00000100:00080000:3.0:1775766631.275790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766631.275793:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766631.275803:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766631.275812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861417472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766631.275820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766631.275823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766631.275824:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766631.275827:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2359 00000100:00080000:3.0:1775766631.275828:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766631.275830:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766631.275834:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766631.275840:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861417600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766631.275848:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861417344:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766631.275936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766631.276007:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766631/real 1775766631] req@ffffa0e9d0542680 x1862023861417344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766631.276033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861417344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766631.276048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766631.276053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766631.276059:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861417344:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766631.276074:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766631.276109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766631.276127:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766631/real 1775766631] req@ffffa0e9d0541880 x1862023861417472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766631.276147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861417472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766631.276156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766631.276160:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766631.276164:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417472:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766631.276214:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766631.276238:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766631.276252:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766631/real 1775766631] req@ffffa0e9d0541500 x1862023861417600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766631.276265:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861417600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766631.276273:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766631.276277:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766631.276280:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417600:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766631.851717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766632.107672:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766632.875682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766633.131594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766633.899659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766634.155592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766634.923694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766635.179620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766635.947605:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766636.203559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766636.395592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766636.395602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766636.395607:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766636.395618:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2364 00000100:00080000:1.0:1775766636.395649:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766636.395655:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766636.395694:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861417728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766636.395720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766636.395723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766636.395724:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766636.395728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2364 00000100:00080000:1.0:1775766636.395731:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766636.395735:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766636.395745:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766636.395754:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861417728:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766636.395755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861417856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766636.395767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766636.395769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766636.395770:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766636.395772:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2364 00000100:00080000:1.0:1775766636.395774:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766636.395777:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766636.395780:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766636.395787:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861417984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766636.395831:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766636.395871:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766636/real 1775766636] req@ffffa0e9c8deea00 x1862023861417728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766636.395887:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861417728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766636.395896:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766636.395899:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766636.395902:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861417728:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766636.395924:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766636.395960:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766636.395970:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766636/real 1775766636] req@ffffa0e9c8dee680 x1862023861417856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766636.395987:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861417856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766636.396004:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766636.396006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766636.396009:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417856:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766636.396013:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766636.396027:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766636.396033:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766636/real 1775766636] req@ffffa0e9c8ded880 x1862023861417984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766636.396042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861417984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766636.396046:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766636.396055:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766636.396058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861417984:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766636.971760:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766637.227610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766637.995628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766638.251621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766639.019621:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766639.275657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766640.043702:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766640.299555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766641.067691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766641.323606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766641.515627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766641.515638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766641.515664:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766641.515676:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2370 00000100:00080000:3.0:1775766641.515689:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766641.515713:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766641.515754:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861418112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766641.515785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766641.515788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766641.515790:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766641.515793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2370 00000100:00080000:3.0:1775766641.515796:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766641.515799:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766641.515808:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766641.515817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861418240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766641.515825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766641.515828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766641.515829:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766641.515832:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2370 00000100:00080000:3.0:1775766641.515834:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766641.515836:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766641.515840:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766641.515846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861418368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766641.515902:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861418112:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766641.515977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766641.516028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766641/real 1775766641] req@ffffa0e9d0542a00 x1862023861418112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766641.516046:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861418112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766641.516069:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766641.516073:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766641.516077:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861418112:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766641.516084:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766641.516104:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766641.516114:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766641/real 1775766641] req@ffffa0e9d0540000 x1862023861418240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766641.516123:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861418240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766641.516129:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766641.516132:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766641.516134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418240:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766641.516138:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766641.516148:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766641.516154:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766641/real 1775766641] req@ffffa0e9d0543b80 x1862023861418368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766641.516162:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861418368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766641.516186:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766641.516188:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766641.516191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418368:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766642.091648:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766642.347600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766643.115677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766643.371615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766644.139694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766644.395593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766645.163620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766645.419591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766646.187710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766646.443591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766646.635678:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766646.635688:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766646.635703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766646.635712:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2375 00000100:00080000:0.0:1775766646.635721:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766646.635727:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766646.635775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861418496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766646.635801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766646.635804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766646.635805:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766646.635808:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2375 00000100:00080000:0.0:1775766646.635810:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766646.635812:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766646.635819:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766646.635843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861418624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766646.635850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766646.635853:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766646.635854:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766646.635856:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2375 00000100:00080000:0.0:1775766646.635860:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766646.635862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766646.635896:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766646.635907:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861418752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766646.635928:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861418496:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766646.636033:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766646.636109:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766646/real 1775766646] req@ffffa0e9d09ded80 x1862023861418496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766646.636141:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861418496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766646.636159:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766646.636166:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766646.636187:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861418496:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766646.636196:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766646.636217:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766646.636227:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766646/real 1775766646] req@ffffa0e9d09dfb80 x1862023861418624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766646.636235:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861418624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766646.636241:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766646.636244:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766646.636246:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418624:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766646.636268:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766646.636296:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766646.636303:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766646/real 1775766646] req@ffffa0e9d09df100 x1862023861418752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766646.636311:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861418752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766646.636316:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766646.636319:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766646.636321:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861418752:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766647.211739:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766647.467605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766648.235639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766648.491681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766649.259674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766649.515639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766650.283872:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766650.539632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766651.307615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766651.563628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766651.755615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766651.755625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766651.755629:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766651.755638:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2380 00000100:00080000:1.0:1775766651.755650:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766651.755656:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766651.755713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861418880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766651.755744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766651.755747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766651.755749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766651.755752:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2380 00000100:00080000:1.0:1775766651.755754:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766651.755757:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766651.755800:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766651.755821:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861418880:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766651.755832:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861419008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766651.755843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766651.755846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766651.755847:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766651.755850:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2380 00000100:00080000:1.0:1775766651.755854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766651.755857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766651.755861:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766651.755871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861419136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766651.755932:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766651.755999:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766651/real 1775766651] req@ffffa0e9c8deca80 x1862023861418880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766651.756023:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861418880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.756035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.756040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766651.756044:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861418880:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766651.756073:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766651.756095:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766651.756107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766651/real 1775766651] req@ffffa0e9c8dedf80 x1862023861419008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766651.756120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861419008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.756130:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.756133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766651.756137:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419008:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766651.756143:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766651.756174:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766651.756184:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766651/real 1775766651] req@ffffa0e9c8dece00 x1862023861419136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766651.756197:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861419136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.756205:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.756209:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766651.756212:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419136:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766652.331723:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766652.587614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766653.355705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766653.611642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766654.379619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766654.635627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766655.403618:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766655.659609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766656.427676:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766656.683630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766656.875617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766656.875628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766656.875633:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766656.875644:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2385 00000100:00080000:1.0:1775766656.875657:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766656.875664:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766656.875709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861419264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766656.875755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766656.875759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766656.875776:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766656.875780:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2385 00000100:00080000:1.0:1775766656.875784:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766656.875788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766656.875799:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766656.875811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861419392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766656.875822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766656.875824:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861419264:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766656.875826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766656.875828:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766656.875836:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2385 00000100:00080000:1.0:1775766656.875839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766656.875842:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766656.875845:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766656.875854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861419520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766656.875929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766656.875974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766656/real 1775766656] req@ffffa0e9c8dec700 x1862023861419264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766656.875993:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861419264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766656.876002:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766656.876006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766656.876010:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861419264:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766656.876032:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766656.876051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766656.876060:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766656/real 1775766656] req@ffffa0e9c8defb80 x1862023861419392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766656.876084:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861419392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766656.876091:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766656.876094:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766656.876096:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419392:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766656.876100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766656.876115:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766656.876122:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766656/real 1775766656] req@ffffa0e9c8def100 x1862023861419520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766656.876131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861419520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766656.876137:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766656.876139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766656.876141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419520:192.168.202.104@tcp:38:kworker.0 00010000:00010000:3.0:1775766657.451663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766657.707574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766658.475687:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766658.731596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766659.499709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766659.755620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:3.0:1775766660.523615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766660.779625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766661.547635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766661.803599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766661.995618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766661.995630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766661.995636:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766661.995649:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2390 00000100:00080000:0.0:1775766661.995674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766661.995683:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766661.995727:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861419648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766661.995766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766661.995771:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766661.995773:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766661.995777:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2390 00000100:00080000:0.0:1775766661.995782:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766661.995787:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766661.995798:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766661.995805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861419648:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766661.995811:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861419776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766661.995822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766661.995826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766661.995827:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766661.995831:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2390 00000100:00080000:0.0:1775766661.995834:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766661.995838:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766661.995843:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766661.995851:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861419904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766661.995882:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766661.995933:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766661/real 1775766661] req@ffffa0e9d09dce00 x1862023861419648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766661.995956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861419648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766661.995974:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766661.995992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766661.995997:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861419648:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766661.996023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766661.996053:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766661.996065:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766661/real 1775766661] req@ffffa0e9d09dc380 x1862023861419776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766661.996075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861419776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766661.996082:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766661.996085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766661.996088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419776:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766661.996092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766661.996106:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766661.996113:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766661/real 1775766661] req@ffffa0e9d09dc000 x1862023861419904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766661.996122:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861419904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766661.996128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766661.996130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766661.996133:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861419904:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766662.571713:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766662.827590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766663.595645:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766663.851584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766664.619728:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766664.875682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766665.643741:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766665.899603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766666.667695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766666.923634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766667.115583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766667.115592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766667.115597:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766667.115607:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2395 00000100:00080000:3.0:1775766667.115634:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766667.115643:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766667.115691:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861420032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766667.115719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766667.115722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766667.115723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766667.115726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2395 00000100:00080000:3.0:1775766667.115729:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766667.115732:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766667.115742:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766667.115754:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861420160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766667.115761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766667.115778:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861420032:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766667.115783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766667.115787:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766667.115793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2395 00000100:00080000:3.0:1775766667.115798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766667.115802:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766667.115809:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766667.115824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861420288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766667.115852:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766667.115901:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766667/real 1775766667] req@ffffa0e9d0541c00 x1862023861420032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766667.115918:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861420032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766667.115926:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766667.115930:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766667.115933:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861420032:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766667.115939:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766667.115954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766667.115962:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766667/real 1775766667] req@ffffa0e9d0540380 x1862023861420160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766667.115970:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861420160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766667.115975:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766667.115977:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766667.115979:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420160:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766667.116000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766667.116011:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766667.116026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766667/real 1775766667] req@ffffa0e9d0542300 x1862023861420288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766667.116034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861420288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766667.116040:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766667.116042:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766667.116044:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420288:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766667.691652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766667.947597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766668.715656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766668.971625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766669.739732:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766669.995606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766670.763704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766671.019626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766671.787761:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766672.043615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766672.235741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766672.235752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766672.235758:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766672.235770:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2400 00000100:00080000:1.0:1775766672.235785:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766672.235793:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766672.235845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861420416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766672.235881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766672.235886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766672.235889:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766672.235895:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2400 00000100:00080000:1.0:1775766672.235900:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766672.235906:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766672.235921:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766672.235961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861420416:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766672.235964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861420544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766672.235990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766672.235993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766672.235995:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766672.236001:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2400 00000100:00080000:1.0:1775766672.236006:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766672.236010:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766672.236016:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766672.236030:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861420672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766672.236075:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766672.236150:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766672/real 1775766672] req@ffffa0e9c8def100 x1862023861420416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766672.236203:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861420416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766672.236220:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766672.236226:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766672.236232:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861420416:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766672.236267:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766672.236303:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766672.236324:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766672/real 1775766672] req@ffffa0e9c8defb80 x1862023861420544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766672.236342:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861420544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766672.236355:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766672.236359:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766672.236364:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420544:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766672.236373:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766672.236402:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766672.236417:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766672/real 1775766672] req@ffffa0e9c8dec700 x1862023861420672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766672.236433:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861420672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766672.236478:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766672.236483:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766672.236530:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420672:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766672.811705:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766673.067654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766673.835711:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766674.091618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766674.859664:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766675.115657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766675.883752:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766676.139646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766676.907732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766677.163584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766677.355617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766677.355628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766677.355633:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766677.355642:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2405 00000100:00080000:3.0:1775766677.355652:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766677.355658:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766677.355698:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861420800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766677.355726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766677.355729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766677.355731:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766677.355734:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2405 00000100:00080000:3.0:1775766677.355736:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766677.355739:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766677.355748:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766677.355776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861420928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766677.355806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766677.355811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766677.355813:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766677.355820:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2405 00000100:00100000:0.0:1775766677.355822:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861420800:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766677.355825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766677.355839:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766677.355843:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766677.355855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861421056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766677.355917:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766677.355970:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766677/real 1775766677] req@ffffa0e9d0543800 x1862023861420800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766677.355988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861420800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766677.355997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766677.356000:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766677.356004:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861420800:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766677.356025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766677.356041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766677.356050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766677/real 1775766677] req@ffffa0e9d0540700 x1862023861420928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766677.356058:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861420928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766677.356063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766677.356066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766677.356068:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861420928:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766677.356087:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766677.356102:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766677.356108:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766677/real 1775766677] req@ffffa0e9d0540e00 x1862023861421056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766677.356116:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861421056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766677.356121:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766677.356123:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766677.356125:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421056:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766677.931695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766678.187653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766678.955706:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766679.211612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766679.979709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766680.235663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766681.003692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766681.259684:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766682.027788:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766682.283686:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766682.475658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766682.475676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766682.475682:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766682.475697:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2410 00000100:00080000:3.0:1775766682.475716:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766682.475728:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766682.475806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861421184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766682.475850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766682.475856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766682.475859:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766682.475866:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2410 00000100:00080000:3.0:1775766682.475873:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766682.475881:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766682.475922:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861421184:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775766682.475922:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766682.475959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861421312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766682.475976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766682.475981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766682.475983:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766682.475989:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2410 00000100:00080000:3.0:1775766682.475995:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766682.475999:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766682.476006:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766682.476021:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861421440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766682.476030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766682.476100:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766682/real 1775766682] req@ffffa0e9d0540a80 x1862023861421184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766682.476127:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861421184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766682.476154:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766682.476162:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766682.476168:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861421184:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766682.476207:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766682.476247:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766682.476263:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766682/real 1775766682] req@ffffa0e9d0542d80 x1862023861421312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766682.476288:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861421312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766682.476298:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766682.476302:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766682.476306:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421312:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766682.476312:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766682.476341:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766682.476359:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766682/real 1775766682] req@ffffa0e9d0541f80 x1862023861421440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766682.476398:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861421440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766682.476413:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766682.476425:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766682.476429:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421440:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766683.051686:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766683.307609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766684.075724:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766684.331636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766685.099628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766685.355580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766686.123644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766686.379578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766687.147595:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766687.403576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766687.595601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766687.595610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766687.595616:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766687.595626:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2416 00000100:00080000:0.0:1775766687.595637:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766687.595643:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766687.595686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861421568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766687.595763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766687.595768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766687.595771:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766687.595776:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2416 00000100:00080000:0.0:1775766687.595797:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766687.595802:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766687.595814:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766687.595839:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861421696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766687.595849:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766687.595853:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766687.595855:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766687.595859:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2416 00000100:00080000:0.0:1775766687.595862:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766687.595866:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766687.595871:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766687.595879:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861421568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766687.595885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861421824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766687.595983:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766687.596044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766687/real 1775766687] req@ffffa0e9d09df480 x1862023861421568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766687.596081:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861421568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766687.596090:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766687.596094:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766687.596104:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861421568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766687.596110:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766687.596129:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766687.596139:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766687/real 1775766687] req@ffffa0e9d09de300 x1862023861421696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766687.596147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861421696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766687.596153:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766687.596155:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766687.596157:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421696:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766687.596177:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766687.596188:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766687.596194:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766687/real 1775766687] req@ffffa0e9d09dea00 x1862023861421824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766687.596202:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861421824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766687.596207:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766687.596209:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766687.596211:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861421824:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766688.171701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766688.427566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766689.195748:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766689.451585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766690.219684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766690.475604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766691.243672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766691.499583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766692.267651:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766692.523667:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766692.715581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766692.715590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766692.715595:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766692.715606:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2421 00000100:00080000:0.0:1775766692.715639:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766692.715646:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766692.715689:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861421952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766692.715718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766692.715722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766692.715723:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766692.715727:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2421 00000100:00080000:0.0:1775766692.715729:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766692.715732:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766692.715742:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766692.715751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861422080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766692.715764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766692.715766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766692.715768:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766692.715771:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2421 00000100:00080000:0.0:1775766692.715773:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766692.715776:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766692.715780:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766692.715819:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861422208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766692.715847:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861421952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766692.715997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422080:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766692.716040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422208:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766693.291665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766693.547559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766694.315663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766694.571628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766695.339701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766695.595616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766695.787807:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766695.787852:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766695.787871:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766695.787921:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766695.787935:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775766695.787988:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766692/real 1775766695] req@ffffa0e9d09df480 x1862023861422208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766695.788034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861422208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766695.788048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766695.788055:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766695.788060:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422208:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766695.788076:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766692/real 1775766695] req@ffffa0e9d09dea00 x1862023861421952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766695.788092:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861421952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766695.788101:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766695.788105:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766695.788109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861421952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766695.788117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766692/real 1775766695] req@ffffa0e9d09de300 x1862023861422080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766695.788130:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861422080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766695.788176:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766695.788181:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766695.788184:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766696.363723:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766696.619561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766697.387649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766697.643610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766697.835588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766697.835598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766697.835603:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766697.835613:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2426 00000100:00080000:3.0:1775766697.835626:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766697.835635:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766697.835692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861422336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766697.835727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766697.835732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766697.835734:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766697.835738:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2426 00000100:00080000:3.0:1775766697.835760:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766697.835770:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766697.835784:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861422336:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775766697.835786:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766697.835817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861422464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766697.835831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766697.835835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766697.835837:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766697.835843:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2426 00000100:00080000:3.0:1775766697.835857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766697.835862:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766697.835869:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766697.835883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861422592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766697.835907:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766697.835965:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766697/real 1775766697] req@ffffa0e9d0541180 x1862023861422336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766697.836004:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861422336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766697.836017:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766697.836022:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766697.836026:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861422336:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766697.836056:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766697.836084:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766697.836098:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766697/real 1775766697] req@ffffa0e9d0543480 x1862023861422464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766697.836124:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861422464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766697.836135:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766697.836139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766697.836143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422464:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766697.836159:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766697.836194:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766697.836207:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766697/real 1775766697] req@ffffa0e9d0543100 x1862023861422592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766697.836221:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861422592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766697.836231:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766697.836235:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766697.836238:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422592:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766698.411654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766698.667595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766699.435624:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766699.691606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766700.459729:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766700.715583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766701.483673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766701.739595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766702.507673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766702.763607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766702.955608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766702.955632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766702.955637:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766702.955646:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2431 00000100:00080000:0.0:1775766702.955659:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766702.955666:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766702.955707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861422720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766702.955751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766702.955759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766702.955761:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766702.955764:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2431 00000100:00080000:0.0:1775766702.955767:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766702.955772:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766702.955783:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766702.955795:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861422848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766702.955800:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861422720:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766702.955813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766702.955817:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766702.955818:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766702.955822:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2431 00000100:00080000:0.0:1775766702.955825:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766702.955828:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766702.955832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766702.955843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861422976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766702.955883:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766702.955935:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766702/real 1775766702] req@ffffa0e9d09dc000 x1862023861422720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766702.955954:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861422720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766702.955965:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766702.955968:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766702.955972:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861422720:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766702.955994:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766702.956039:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766702.956050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766702/real 1775766702] req@ffffa0e9d09dc380 x1862023861422848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766702.956060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861422848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766702.956066:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766702.956069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766702.956072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422848:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766702.956076:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766702.956088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766702.956094:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766702/real 1775766702] req@ffffa0e9d09dce00 x1862023861422976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766702.956102:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861422976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766702.956108:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766702.956110:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766702.956112:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861422976:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766703.531654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766703.787592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766704.555768:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766704.811577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766705.579665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766705.835611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766706.603671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766706.859608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766707.627698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766707.883583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766708.075559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766708.075567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766708.075572:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766708.075582:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2436 00000100:00080000:0.0:1775766708.075594:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766708.075601:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766708.075652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861423104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766708.075677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766708.075681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766708.075682:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766708.075685:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2436 00000100:00080000:0.0:1775766708.075688:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766708.075691:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766708.075700:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766708.075709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861423232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766708.075718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766708.075721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766708.075722:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766708.075727:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2436 00000100:00100000:1.0:1775766708.075729:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861423104:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766708.075730:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766708.075734:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766708.075752:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766708.075762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861423360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766708.075805:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766708.075850:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766708/real 1775766708] req@ffffa0e9d09df100 x1862023861423104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766708.075874:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861423104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766708.075884:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766708.075887:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766708.075891:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861423104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766708.075920:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766708.075939:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766708.075948:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766708/real 1775766708] req@ffffa0e9d09dfb80 x1862023861423232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766708.075956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861423232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766708.075962:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766708.075965:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766708.075967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766708.075971:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766708.075981:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766708.075987:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766708/real 1775766708] req@ffffa0e9d09ded80 x1862023861423360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766708.075995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861423360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766708.076009:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766708.076012:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766708.076014:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423360:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766708.651783:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766708.907550:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766709.675690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766709.931607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766710.699729:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766710.955545:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766711.723740:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766711.979590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766712.747733:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766713.003652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766713.195638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766713.195648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766713.195654:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766713.195666:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2441 00000100:00080000:0.0:1775766713.195679:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766713.195686:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766713.195731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861423488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766713.195754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766713.195758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766713.195760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766713.195764:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2441 00000100:00080000:0.0:1775766713.195767:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766713.195770:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766713.195791:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766713.195798:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861423488:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766713.195806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861423616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766713.195816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766713.195819:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766713.195821:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766713.195824:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2441 00000100:00080000:0.0:1775766713.195842:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766713.195857:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766713.195863:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766713.195873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861423744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766713.195903:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766713.195960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766713/real 1775766713] req@ffffa0e9d09dc700 x1862023861423488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766713.195980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861423488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766713.195991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766713.195995:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766713.195999:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861423488:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766713.196028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766713.196057:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766713.196073:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766713/real 1775766713] req@ffffa0e9d09df800 x1862023861423616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766713.196088:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861423616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766713.196095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766713.196098:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766713.196110:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423616:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766713.196115:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766713.196154:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766713.196186:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766713/real 1775766713] req@ffffa0e9d09dd880 x1862023861423744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766713.196208:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861423744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766713.196223:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766713.196229:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766713.196235:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861423744:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766713.771728:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766714.027563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766714.795669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766715.051583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766715.819679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766716.075612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766716.843617:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766717.099605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766717.867816:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766718.123547:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766718.315613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766718.315622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766718.315628:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766718.315639:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2446 00000100:00080000:1.0:1775766718.315652:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766718.315659:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766718.315704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861423872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766718.315744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766718.315747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766718.315748:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766718.315766:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2446 00000100:00080000:1.0:1775766718.315769:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766718.315773:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766718.315783:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766718.315792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861424000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766718.315800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766718.315803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766718.315805:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766718.315808:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2446 00000100:00080000:1.0:1775766718.315810:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766718.315813:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766718.315816:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766718.315840:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861423872:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766718.315841:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861424128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766718.315952:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766718.316053:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766718/real 1775766718] req@ffffa0e9c8dece00 x1862023861423872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.316117:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861423872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.316126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.316130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766718.316133:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861423872:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766718.316140:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766718.316160:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766718.316169:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766718/real 1775766718] req@ffffa0e9c8dedf80 x1862023861424000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.316178:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861424000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.316184:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.316186:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766718.316196:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424000:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766718.316219:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766718.316234:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766718.316240:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766718/real 1775766718] req@ffffa0e9c8deca80 x1862023861424128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.316248:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861424128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.316253:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.316255:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766718.316258:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424128:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766718.891710:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766719.147634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766719.915713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766720.171611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766720.939653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766721.195573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766721.963620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766722.219581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766722.987769:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766723.243605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766723.435621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766723.435634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766723.435639:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766723.435649:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2451 00000100:00080000:1.0:1775766723.435661:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766723.435668:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766723.435710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861424256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766723.435765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766723.435769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766723.435771:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766723.435775:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2451 00000100:00080000:1.0:1775766723.435780:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766723.435784:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766723.435794:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766723.435809:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861424384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766723.435814:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861424256:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766723.435828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766723.435843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766723.435844:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766723.435849:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2451 00000100:00080000:1.0:1775766723.435854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766723.435858:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766723.435864:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766723.435876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861424512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766723.435891:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766723.435945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766723/real 1775766723] req@ffffa0e9c8ded880 x1862023861424256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766723.435960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861424256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766723.435969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766723.435973:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766723.435976:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861424256:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766723.435994:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766723.436009:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766723.436041:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766723/real 1775766723] req@ffffa0e9c8dee680 x1862023861424384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766723.436051:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861424384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766723.436067:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766723.436069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766723.436071:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424384:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766723.436076:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766723.436089:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766723.436096:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766723/real 1775766723] req@ffffa0e9c8deea00 x1862023861424512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766723.436103:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861424512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766723.436108:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766723.436110:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766723.436111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424512:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766724.011616:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766724.267582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766725.035656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766725.291588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766726.059691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766726.315681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766727.083693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766727.339644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766728.107674:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766728.363574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766728.555664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766728.555674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766728.555679:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766728.555689:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2457 00000100:00080000:1.0:1775766728.555726:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766728.555733:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766728.555769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861424640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766728.555802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766728.555805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766728.555807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766728.555810:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2457 00000100:00080000:1.0:1775766728.555813:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766728.555816:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766728.555825:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766728.555834:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861424768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766728.555843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766728.555849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766728.555851:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766728.555856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2457 00000100:00080000:1.0:1775766728.555861:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766728.555866:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766728.555873:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766728.555899:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861424896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766728.555941:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861424640:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766728.556037:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766728.556087:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766728/real 1775766728] req@ffffa0e9c8dedc00 x1862023861424640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766728.556106:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861424640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766728.556133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766728.556139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766728.556144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861424640:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766728.556154:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766728.556181:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766728.556194:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766728/real 1775766728] req@ffffa0e9c8def480 x1862023861424768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766728.556204:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861424768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766728.556210:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766728.556212:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766728.556215:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424768:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766728.556220:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766728.556233:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766728.556240:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766728/real 1775766728] req@ffffa0e9c8def800 x1862023861424896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766728.556247:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861424896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766728.556253:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766728.556255:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766728.556257:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861424896:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766729.131694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766729.387593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766730.155706:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766730.411554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766731.179707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766731.435598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766732.203692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766732.459608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766733.227687:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766733.483605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766733.675646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766733.675656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766733.675660:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766733.675671:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2462 00000100:00080000:3.0:1775766733.675684:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766733.675707:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766733.675759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861425024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766733.675792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766733.675811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766733.675814:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766733.675818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2462 00000100:00080000:3.0:1775766733.675822:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766733.675827:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766733.675854:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766733.675866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861425152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766733.675876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766733.675879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766733.675882:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766733.675890:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861425024:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766733.675891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2462 00000100:00080000:3.0:1775766733.675904:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766733.675911:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766733.675937:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766733.675951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861425280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766733.675988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766733.676037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766733/real 1775766733] req@ffffa0e9d0543100 x1862023861425024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766733.676052:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861425024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766733.676061:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766733.676066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766733.676070:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861425024:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766733.676092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766733.676108:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766733.676116:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766733/real 1775766733] req@ffffa0e9d0543480 x1862023861425152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766733.676128:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861425152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766733.676136:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766733.676140:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766733.676143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425152:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766733.676150:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766733.676174:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766733.676186:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766733/real 1775766733] req@ffffa0e9d0541180 x1862023861425280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766733.676207:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861425280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766733.676213:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766733.676215:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766733.676217:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425280:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766734.251854:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766734.507610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766735.275738:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766735.531577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766736.299691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766736.555592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766737.323666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766737.579616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766738.347753:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766738.603592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766738.795624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766738.795634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766738.795639:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766738.795649:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2467 00000100:00080000:0.0:1775766738.795661:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766738.795668:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766738.795713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861425408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766738.795747:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766738.795750:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766738.795752:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766738.795761:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2467 00000100:00080000:0.0:1775766738.795765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766738.795769:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766738.795779:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766738.795794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861425536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766738.795797:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861425408:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766738.795839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766738.795843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766738.795845:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766738.795850:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2467 00000100:00080000:0.0:1775766738.795855:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766738.795859:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766738.795891:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766738.795905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861425664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766738.795918:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766738.795982:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766738/real 1775766738] req@ffffa0e9d09dd500 x1862023861425408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766738.795998:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861425408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766738.796022:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766738.796025:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766738.796028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861425408:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766738.796049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766738.796077:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766738.796085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766738/real 1775766738] req@ffffa0e9d09dd180 x1862023861425536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766738.796092:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861425536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766738.796098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766738.796100:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766738.796102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425536:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766738.796106:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766738.796144:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766738.796151:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766738/real 1775766738] req@ffffa0e9d09ddc00 x1862023861425664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766738.796159:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861425664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766738.796164:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766738.796165:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766738.796167:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425664:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766739.371644:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766739.627599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766740.395655:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766740.651632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766741.419682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766741.675617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766742.443624:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766742.699642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766743.467632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766743.723583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766743.915589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766743.915602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766743.915608:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766743.915619:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2472 00000100:00080000:2.0:1775766743.915631:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766743.915639:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766743.915681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861425792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766743.915708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766743.915712:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766743.915714:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766743.915717:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2472 00000100:00080000:2.0:1775766743.915722:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766743.915726:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766743.915756:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766743.915771:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861425792:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766743.915779:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861425920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766743.915793:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766743.915795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766743.915796:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766743.915799:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2472 00000100:00080000:2.0:1775766743.915802:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766743.915805:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766743.915825:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766743.915835:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861426048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766743.915884:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766743.915941:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766743/real 1775766743] req@ffffa0e9d0710a80 x1862023861425792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766743.915960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861425792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766743.915971:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766743.915975:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766743.915978:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861425792:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766743.916002:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766743.916025:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766743.916039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766743/real 1775766743] req@ffffa0e9d0710e00 x1862023861425920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766743.916076:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861425920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766743.916083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766743.916086:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766743.916088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861425920:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766743.916093:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766743.916110:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766743.916118:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766743/real 1775766743] req@ffffa0e9d0712a00 x1862023861426048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766743.916127:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861426048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766743.916132:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766743.916134:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766743.916136:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426048:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775766744.491676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766744.747569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766745.515632:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766745.771579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766746.539667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766746.795683:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775766747.563713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766747.819565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766748.587675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766748.843600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766749.035589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766749.035598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766749.035603:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766749.035614:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2477 00000100:00080000:3.0:1775766749.035626:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766749.035632:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766749.035682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861426176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766749.035721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766749.035726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766749.035728:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766749.035733:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2477 00000100:00080000:3.0:1775766749.035738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766749.035743:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766749.035766:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766749.035777:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861426176:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766749.035781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861426304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766749.035792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766749.035795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766749.035797:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766749.035801:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2477 00000100:00080000:3.0:1775766749.035804:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766749.035808:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766749.035813:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766749.035821:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861426432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766749.035853:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766749.035906:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766749/real 1775766749] req@ffffa0e9d0541f80 x1862023861426176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766749.035924:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861426176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766749.035939:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766749.035943:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766749.035947:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861426176:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766749.035971:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766749.035990:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766749.036000:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766749/real 1775766749] req@ffffa0e9d0542d80 x1862023861426304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766749.036008:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861426304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766749.036014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766749.036016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766749.036018:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426304:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766749.036022:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766749.036032:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766749.036038:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766749/real 1775766749] req@ffffa0e9d0540a80 x1862023861426432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766749.036045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861426432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766749.036051:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766749.036052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766749.036054:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426432:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766749.611733:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766749.867555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766750.635666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766750.891578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766751.659612:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766751.915608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766752.683821:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766752.939638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766753.707779:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766753.963606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766754.155619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766754.155631:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766754.155636:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766754.155649:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2482 00000100:00080000:3.0:1775766754.155680:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766754.155688:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766754.155755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861426560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766754.155785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766754.155789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766754.155790:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766754.155794:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2482 00000100:00080000:3.0:1775766754.155798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766754.155802:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766754.155814:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766754.155827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861426688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766754.155836:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861426560:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766754.155846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766754.155850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766754.155852:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766754.155857:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2482 00000100:00080000:3.0:1775766754.155861:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766754.155865:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766754.155870:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766754.155883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861426816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766754.155916:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766754.155968:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766754/real 1775766754] req@ffffa0e9d0540e00 x1862023861426560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766754.155985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861426560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766754.155995:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766754.155999:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766754.156002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861426560:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766754.156023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766754.156039:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766754.156047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766754/real 1775766754] req@ffffa0e9d0540700 x1862023861426688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766754.156054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861426688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766754.156060:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766754.156063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766754.156065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426688:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766754.156069:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766754.156078:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766754.156084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766754/real 1775766754] req@ffffa0e9d0543800 x1862023861426816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766754.156099:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861426816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766754.156105:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766754.156107:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766754.156109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861426816:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766754.731682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766754.987601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766755.755660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766756.011632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766756.779701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766757.035612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766757.803688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766758.059679:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766758.827670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766759.083596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766759.275573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766759.275616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766759.275621:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766759.275632:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2487 00000100:00080000:3.0:1775766759.275660:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766759.275685:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766759.275725:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861426944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766759.275749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766759.275752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766759.275754:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766759.275758:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2487 00000100:00080000:3.0:1775766759.275761:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766759.275763:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766759.275775:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766759.275784:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861427072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766759.275793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766759.275798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766759.275800:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766759.275801:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861426944:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766759.275805:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2487 00000100:00080000:3.0:1775766759.275811:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766759.275815:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766759.275818:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766759.275827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861427200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766759.275954:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427072:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766759.275998:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427200:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766759.851674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766760.107632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766760.875657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766761.131645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766761.899659:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766762.155672:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775766762.347838:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775766762.347882:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766762.347899:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766762.347939:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766762.347946:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775766762.348030:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766759/real 1775766762] req@ffffa0e9d0540380 x1862023861427072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766762.348070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861427072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766762.348085:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766762.348091:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766762.348096:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427072:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766762.348107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766759/real 1775766762] req@ffffa0e9d0541c00 x1862023861427200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766762.348121:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861427200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766762.348161:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766762.348166:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766762.348170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766762.348178:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766759/real 1775766762] req@ffffa0e9d0542300 x1862023861426944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766762.348193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861426944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766762.348201:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766762.348204:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766762.348208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861426944:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775766762.923674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766763.179660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766763.947727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766764.203641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766764.395649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766764.395661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766764.395667:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766764.395681:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2492 00000100:00080000:1.0:1775766764.395696:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766764.395705:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766764.395762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861427328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766764.395800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766764.395805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766764.395807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766764.395812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2492 00000100:00080000:1.0:1775766764.395834:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766764.395841:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766764.395851:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861427328:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775766764.395858:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766764.395880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861427456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766764.395895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766764.395898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766764.395901:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766764.395907:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2492 00000100:00080000:1.0:1775766764.395912:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766764.395918:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766764.395924:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766764.395937:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861427584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766764.395951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766764.396010:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766764/real 1775766764] req@ffffa0e9c8dec380 x1862023861427328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766764.396033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861427328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766764.396047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766764.396052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766764.396057:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861427328:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766764.396089:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766764.396119:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766764.396134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766764/real 1775766764] req@ffffa0e9c8ded500 x1862023861427456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766764.396166:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861427456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766764.396174:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766764.396179:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766764.396182:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427456:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766764.396189:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766764.396212:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766764.396224:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766764/real 1775766764] req@ffffa0e9c8deed80 x1862023861427584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766764.396235:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861427584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766764.396242:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766764.396246:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766764.396249:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427584:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766764.971742:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766765.227566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766765.995693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766766.251608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766767.019715:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766767.275663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766768.043723:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766768.299594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766769.067686:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766769.323641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766769.515696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766769.515708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766769.515714:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766769.515727:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2498 00000100:00080000:1.0:1775766769.515741:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766769.515755:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766769.515800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861427712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766769.515847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766769.515869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766769.515872:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766769.515877:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2498 00000100:00080000:1.0:1775766769.515883:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766769.515889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766769.515902:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766769.515922:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861427712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766769.515922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861427840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766769.515941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766769.515945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766769.515947:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766769.515952:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2498 00000100:00080000:1.0:1775766769.515957:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766769.515962:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766769.515968:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766769.515982:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861427968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766769.516036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766769.516109:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766769/real 1775766769] req@ffffa0e9c8dee300 x1862023861427712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766769.516128:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861427712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766769.516149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766769.516153:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766769.516156:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861427712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766769.516179:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766769.516199:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766769.516208:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766769/real 1775766769] req@ffffa0e9c8dec000 x1862023861427840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766769.516217:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861427840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766769.516223:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766769.516225:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766769.516227:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427840:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766769.516231:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766769.516241:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766769.516247:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766769/real 1775766769] req@ffffa0e9c8ded180 x1862023861427968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766769.516254:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861427968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766769.516260:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766769.516261:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766769.516263:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861427968:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766770.091647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766770.347614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766771.115738:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766771.371626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766772.139707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766772.395676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766773.163797:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766773.419637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766774.187696:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766774.443575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766774.635612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766774.635621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766774.635625:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766774.635634:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2503 00000100:00080000:1.0:1775766774.635645:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766774.635651:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766774.635697:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861428096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766774.635734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766774.635737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766774.635738:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766774.635741:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2503 00000100:00080000:1.0:1775766774.635745:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766774.635749:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766774.635758:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766774.635771:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861428224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766774.635777:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861428096:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766774.635786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766774.635790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766774.635792:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766774.635795:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2503 00000100:00080000:1.0:1775766774.635797:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766774.635799:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766774.635803:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766774.635813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861428352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766774.635867:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766774.635915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766774/real 1775766774] req@ffffa0e9c8ded180 x1862023861428096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766774.635937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861428096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766774.635946:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766774.635950:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766774.635953:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861428096:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766774.635976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428224:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766774.635992:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766774.636000:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766774/real 1775766774] req@ffffa0e9c8dec000 x1862023861428224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766774.636008:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861428224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766774.636013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766774.636016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766774.636018:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766774.636022:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766774.636032:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766774.636037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766774/real 1775766774] req@ffffa0e9c8dee300 x1862023861428352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766774.636059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861428352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766774.636065:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766774.636067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766774.636069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428352:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766775.211712:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766775.467634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766776.235733:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766776.491616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766777.259693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766777.515568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766778.283693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766778.539613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766779.307672:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766779.563619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766779.755609:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766779.755618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766779.755624:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766779.755633:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2508 00000100:00080000:3.0:1775766779.755645:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766779.755651:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766779.755692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861428480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766779.755719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766779.755722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766779.755723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766779.755726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2508 00000100:00080000:3.0:1775766779.755729:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766779.755731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766779.755741:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766779.755750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861428608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766779.755758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766779.755775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766779.755776:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766779.755780:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2508 00000100:00080000:3.0:1775766779.755783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766779.755786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766779.755790:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766779.755792:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861428480:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766779.755800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861428736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766779.755875:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766779.755914:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766779/real 1775766779] req@ffffa0e9d0543b80 x1862023861428480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766779.755931:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861428480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766779.755940:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766779.755943:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766779.755946:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861428480:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766779.755951:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766779.755968:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766779.755976:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766779/real 1775766779] req@ffffa0e9d0540000 x1862023861428608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766779.755984:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861428608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766779.756000:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766779.756003:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766779.756005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428608:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766779.756027:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766779.756041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766779.756048:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766779/real 1775766779] req@ffffa0e9d0542a00 x1862023861428736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766779.756055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861428736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766779.756061:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766779.756063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766779.756065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428736:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766780.331695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766780.587612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766781.355684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766781.611607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766782.379675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766782.635599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766783.403694:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766783.659635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766784.427780:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766784.683621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766784.875592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766784.875601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766784.875606:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766784.875615:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2513 00000100:00080000:3.0:1775766784.875628:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766784.875634:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766784.875678:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861428864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766784.875706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766784.875709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766784.875710:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766784.875713:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2513 00000100:00080000:3.0:1775766784.875716:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766784.875719:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766784.875729:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766784.875738:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861428992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766784.875755:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861428864:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766784.875770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766784.875773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766784.875775:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766784.875779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2513 00000100:00080000:3.0:1775766784.875782:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766784.875785:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766784.875790:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766784.875801:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861429120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766784.875897:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766784.875968:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766784/real 1775766784] req@ffffa0e9d0541500 x1862023861428864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766784.875988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861428864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766784.875998:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766784.876002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766784.876014:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861428864:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766784.876058:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766784.876082:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766784.876093:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766784/real 1775766784] req@ffffa0e9d0541880 x1862023861428992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766784.876103:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861428992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766784.876110:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766784.876113:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766784.876116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861428992:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766784.876120:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766784.876133:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766784.876140:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766784/real 1775766784] req@ffffa0e9d0542680 x1862023861429120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766784.876149:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861429120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766784.876155:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766784.876157:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766784.876160:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429120:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766785.451702:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766785.707603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766786.475693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766786.731632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766787.499655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766787.755616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766788.523691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766788.779598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766789.547699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766789.803585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766789.995634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766789.995644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766789.995671:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766789.995681:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2518 00000100:00080000:0.0:1775766789.995691:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766789.995697:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766789.995733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861429248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766789.995766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766789.995769:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766789.995771:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766789.995774:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2518 00000100:00080000:0.0:1775766789.995777:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766789.995780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766789.995788:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766789.995798:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861429376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766789.995808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766789.995811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766789.995812:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766789.995818:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2518 00000100:00100000:1.0:1775766789.995819:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861429248:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766789.995820:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766789.995828:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766789.995832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766789.995841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861429504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766789.995887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766789.995935:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766789/real 1775766789] req@ffffa0e9d09ddf80 x1862023861429248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766789.995973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861429248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766789.995982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766789.995986:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766789.995989:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861429248:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766789.996011:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766789.996031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766789.996041:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766789/real 1775766789] req@ffffa0e9d09dca80 x1862023861429376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766789.996050:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861429376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766789.996057:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766789.996059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766789.996062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429376:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766789.996066:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766789.996078:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766789.996084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766789/real 1775766789] req@ffffa0e9d09de680 x1862023861429504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766789.996093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861429504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766789.996108:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766789.996111:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766789.996122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429504:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766790.571708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766790.827650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766791.595693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766791.851569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766792.619778:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766792.875595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766793.643744:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766793.899592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766794.667637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766794.923581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766795.115615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766795.115624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766795.115629:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766795.115637:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2523 00000100:00080000:2.0:1775766795.115646:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766795.115652:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766795.115690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861429632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766795.115715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766795.115717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766795.115737:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766795.115739:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2523 00000100:00080000:2.0:1775766795.115742:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766795.115745:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766795.115764:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766795.115772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861429760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766795.115780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766795.115783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766795.115784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766795.115788:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2523 00000100:00080000:2.0:1775766795.115790:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766795.115792:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766795.115795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766795.115801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861429888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766795.115811:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861429632:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766795.115879:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766795.115924:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766795/real 1775766795] req@ffffa0e9d0713480 x1862023861429632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766795.115941:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861429632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766795.115949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766795.115952:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766795.115956:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861429632:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766795.115961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766795.115975:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766795.115982:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766795/real 1775766795] req@ffffa0e9d0711500 x1862023861429760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766795.115990:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861429760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766795.115995:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766795.115997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766795.115999:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429760:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766795.116028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766795.116041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766795.116047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766795/real 1775766795] req@ffffa0e9d0711880 x1862023861429888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766795.116055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861429888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766795.116060:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766795.116062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766795.116064:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861429888:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766795.691666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766795.947549:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766796.715636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766796.971617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766797.739757:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766797.995607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766798.763704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766799.019627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766799.787636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766800.043634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766800.235665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766800.235679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766800.235686:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766800.235718:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2528 00000100:00080000:2.0:1775766800.235735:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766800.235745:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766800.235803:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861430016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766800.235842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766800.235847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766800.235850:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766800.235854:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2528 00000100:00080000:2.0:1775766800.235858:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766800.235862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766800.235876:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766800.235887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861430144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766800.235898:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766800.235903:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766800.235905:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766800.235910:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2528 00000100:00080000:2.0:1775766800.235914:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766800.235918:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766800.235923:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766800.235962:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861430272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766800.235991:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861430016:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775766800.236095:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766800.236186:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766800/real 1775766800] req@ffffa0e9d0712300 x1862023861430016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766800.236216:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861430016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766800.236266:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766800.236276:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766800.236287:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861430016:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766800.236305:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766800.236354:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766800.236378:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766800/real 1775766800] req@ffffa0e9d0713100 x1862023861430144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766800.236431:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861430144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766800.236447:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766800.236454:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766800.236461:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430144:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766800.236561:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766800.236592:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766800.236607:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766800/real 1775766800] req@ffffa0e9d0711880 x1862023861430272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766800.236622:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861430272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766800.236631:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766800.236635:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766800.236649:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430272:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766800.811711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766801.067637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766801.835713:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766802.091631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766802.859688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766803.115650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766803.883740:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766804.139582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766804.907767:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766805.163579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766805.355658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766805.355672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766805.355678:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766805.355690:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2533 00000100:00080000:3.0:1775766805.355704:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766805.355712:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766805.355766:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861430400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766805.355805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766805.355810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766805.355812:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766805.355816:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2533 00000100:00080000:3.0:1775766805.355820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766805.355824:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766805.355862:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766805.355890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861430528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766805.355905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766805.355908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766805.355911:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766805.355921:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2533 00000100:00100000:1.0:1775766805.355923:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861430400:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766805.355927:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766805.355940:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766805.355946:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766805.355959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861430656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766805.356036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766805.356086:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766805/real 1775766805] req@ffffa0e9d0542680 x1862023861430400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766805.356104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861430400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766805.356149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766805.356154:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766805.356159:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861430400:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766805.356231:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766805.356253:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766805.356263:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766805/real 1775766805] req@ffffa0e9d0541880 x1862023861430528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766805.356272:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861430528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766805.356279:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766805.356281:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766805.356284:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430528:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766805.356288:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766805.356300:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766805.356306:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766805/real 1775766805] req@ffffa0e9d0541500 x1862023861430656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766805.356313:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861430656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766805.356319:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766805.356321:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766805.356323:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430656:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766805.931693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766806.187621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766806.955721:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766807.211613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766807.979766:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766808.235634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766809.003635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766809.259556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766810.027670:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766810.283587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766810.475636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766810.475648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766810.475653:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766810.475665:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2538 00000100:00080000:0.0:1775766810.475681:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766810.475692:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766810.475761:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861430784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766810.475797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766810.475801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766810.475804:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766810.475808:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2538 00000100:00080000:0.0:1775766810.475813:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766810.475818:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766810.475831:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766810.475848:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861430912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766810.475860:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861430784:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766810.475864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766810.475874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766810.475878:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766810.475883:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2538 00000100:00080000:0.0:1775766810.475886:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766810.475890:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766810.475912:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766810.475928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861431040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766810.475972:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766810.476030:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766810/real 1775766810] req@ffffa0e9d09de680 x1862023861430784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766810.476060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861430784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766810.476078:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766810.476084:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766810.476090:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861430784:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766810.476140:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766810.476169:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766810.476183:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766810/real 1775766810] req@ffffa0e9d09dca80 x1862023861430912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766810.476196:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861430912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766810.476205:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766810.476208:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766810.476211:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861430912:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766810.476217:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766810.476233:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766810.476242:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766810/real 1775766810] req@ffffa0e9d09ddf80 x1862023861431040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766810.476264:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861431040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766810.476285:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766810.476288:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766810.476292:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431040:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766811.051817:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766811.307667:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766812.075695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766812.331635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766813.099679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766813.355656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766814.123692:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766814.379598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766815.147702:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766815.403572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766815.595611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766815.595625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766815.595632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766815.595646:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2544 00000100:00080000:0.0:1775766815.595663:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766815.595673:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766815.595730:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861431168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766815.595788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766815.595792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766815.595795:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766815.595800:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2544 00000100:00080000:0.0:1775766815.595804:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766815.595808:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766815.595822:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766815.595838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861431296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766815.595852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766815.595861:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775766815.595862:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861431168:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766815.595866:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766815.595878:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2544 00000100:00080000:0.0:1775766815.595904:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766815.595910:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766815.595918:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766815.595935:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861431424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766815.595968:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766815.596043:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766815/real 1775766815] req@ffffa0e9d09ddc00 x1862023861431168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766815.596093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861431168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766815.596108:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766815.596115:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766815.596123:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861431168:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766815.596163:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766815.596208:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766815.596233:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766815/real 1775766815] req@ffffa0e9d09dd180 x1862023861431296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766815.596257:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861431296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766815.596299:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766815.596306:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766815.596334:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431296:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766815.596347:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766815.596390:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766815.596407:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766815/real 1775766815] req@ffffa0e9d09dd500 x1862023861431424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766815.596422:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861431424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766815.596433:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766815.596437:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766815.596441:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431424:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766816.171744:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766816.427620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766817.195742:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766817.451562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766818.219739:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766818.475596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766819.243658:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766819.499578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766820.267752:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766820.523602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766820.715630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766820.715639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766820.715644:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766820.715655:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2549 00000100:00080000:3.0:1775766820.715668:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766820.715674:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766820.715717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861431552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766820.715743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766820.715746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766820.715747:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766820.715750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2549 00000100:00080000:3.0:1775766820.715768:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766820.715772:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766820.715794:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766820.715806:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861431552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766820.715806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861431680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766820.715823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766820.715825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766820.715827:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766820.715830:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2549 00000100:00080000:3.0:1775766820.715832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766820.715835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766820.715838:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766820.715845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861431808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766820.715899:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766820.715940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766820/real 1775766820] req@ffffa0e9d0542a00 x1862023861431552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766820.715967:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861431552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766820.715976:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766820.715979:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766820.715988:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861431552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766820.716029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766820.716049:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766820.716058:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766820/real 1775766820] req@ffffa0e9d0540000 x1862023861431680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766820.716067:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861431680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766820.716073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766820.716076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766820.716078:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431680:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766820.716082:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766820.716093:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766820.716100:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766820/real 1775766820] req@ffffa0e9d0543b80 x1862023861431808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766820.716112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861431808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766820.716131:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766820.716135:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766820.716139:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861431808:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766821.291658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766821.547583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766822.315798:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766822.571599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766823.339681:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766823.595556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766824.363671:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766824.619600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766825.387752:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766825.643571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766825.835592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766825.835603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766825.835609:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766825.835622:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2554 00000100:00080000:3.0:1775766825.835638:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766825.835646:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766825.835697:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861431936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766825.835730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766825.835750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766825.835752:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766825.835756:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2554 00000100:00080000:3.0:1775766825.835761:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766825.835766:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766825.835780:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766825.835794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861432064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766825.835808:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861431936:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766825.835810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766825.835819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766825.835824:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766825.835829:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2554 00000100:00080000:3.0:1775766825.835832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766825.835835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766825.835840:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766825.835849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861432192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766825.836014:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432064:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766825.836080:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432192:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766826.411638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766826.667641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766827.435682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766827.691613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766828.459594:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766828.715563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766828.907705:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766828.907756:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766828.907771:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766828.907803:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766828.907835:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775766828.907876:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766825/real 1775766828] req@ffffa0e9d0541c00 x1862023861432064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766828.907934:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861432064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766828.907949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766828.907956:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766828.907960:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432064:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766828.907970:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766825/real 1775766828] req@ffffa0e9d0540380 x1862023861432192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766828.907991:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861432192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766828.908000:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766828.908004:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766828.908008:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766828.908020:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766825/real 1775766828] req@ffffa0e9d0542300 x1862023861431936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766828.908034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861431936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766828.908041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766828.908045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766828.908068:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861431936:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775766829.483664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766829.739626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766830.507643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766830.763569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766830.955630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766830.955640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766830.955645:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766830.955655:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2559 00000100:00080000:1.0:1775766830.955667:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766830.955674:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766830.955713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861432320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766830.955743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766830.955746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766830.955747:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766830.955750:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2559 00000100:00080000:1.0:1775766830.955752:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766830.955754:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766830.955771:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766830.955781:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861432448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766830.955788:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861432320:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766830.955791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766830.955797:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766830.955798:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766830.955802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2559 00000100:00080000:1.0:1775766830.955804:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766830.955832:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766830.955837:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766830.955846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861432576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766830.955884:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766830.955940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766830/real 1775766830] req@ffffa0e9c8deed80 x1862023861432320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766830.955957:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861432320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766830.955966:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766830.955970:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766830.955973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861432320:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766830.955998:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766830.956015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766830.956025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766830/real 1775766830] req@ffffa0e9c8ded500 x1862023861432448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766830.956046:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861432448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766830.956054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766830.956061:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766830.956073:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432448:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766830.956081:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766830.956104:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766830.956115:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766830/real 1775766830] req@ffffa0e9c8dec380 x1862023861432576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766830.956129:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861432576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766830.956136:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766830.956140:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766830.956144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432576:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766831.531739:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766831.787670:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766832.555761:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766832.811666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766833.579760:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766833.835649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766834.603756:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766834.859578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766835.627750:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766835.883583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766836.075601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766836.075610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766836.075614:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766836.075624:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2564 00000100:00080000:1.0:1775766836.075633:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766836.075639:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766836.075674:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861432704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766836.075699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766836.075701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766836.075703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766836.075706:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2564 00000100:00080000:1.0:1775766836.075708:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766836.075710:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766836.075719:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766836.075726:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861432832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766836.075733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766836.075736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766836.075758:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766836.075761:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2564 00000100:00080000:1.0:1775766836.075764:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766836.075767:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766836.075771:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766836.075780:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861432960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766836.075803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861432704:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766836.075941:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766836.076005:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766836/real 1775766836] req@ffffa0e9c8def800 x1862023861432704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766836.076029:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861432704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766836.076041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766836.076047:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766836.076051:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861432704:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766836.076060:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766836.076101:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766836.076130:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766836/real 1775766836] req@ffffa0e9c8def480 x1862023861432832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766836.076146:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861432832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766836.076155:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766836.076159:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766836.076163:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432832:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766836.076209:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766836.076230:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766836.076242:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766836/real 1775766836] req@ffffa0e9c8dedc00 x1862023861432960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766836.076254:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861432960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766836.076262:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766836.076266:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766836.076269:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861432960:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766836.651680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766836.907636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766837.675645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766837.931592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766838.699798:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766838.955578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766839.723656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766839.979630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766840.747672:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766841.003608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766841.195654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766841.195664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766841.195669:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766841.195680:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2569 00000100:00080000:3.0:1775766841.195694:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766841.195701:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766841.195745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861433088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766841.195812:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766841.195816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766841.195818:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766841.195821:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2569 00000100:00080000:3.0:1775766841.195825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766841.195829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766841.195840:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766841.195853:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861433216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766841.195862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766841.195866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766841.195867:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766841.195871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2569 00000100:00080000:3.0:1775766841.195876:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766841.195877:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861433088:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766841.195881:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766841.195889:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766841.195896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861433344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766841.195967:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766841.196010:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766841/real 1775766841] req@ffffa0e9d0543800 x1862023861433088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766841.196052:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861433088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766841.196073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766841.196078:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766841.196084:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861433088:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766841.196117:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766841.196183:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766841.196200:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766841/real 1775766841] req@ffffa0e9d0540700 x1862023861433216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766841.196218:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861433216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766841.196229:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766841.196233:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766841.196237:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433216:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766841.196270:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766841.196308:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766841.196320:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766841/real 1775766841] req@ffffa0e9d0540e00 x1862023861433344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766841.196333:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861433344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766841.196341:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766841.196344:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766841.196348:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433344:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766841.771695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766842.027635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766842.795753:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766843.051576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766843.819627:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766844.075578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766844.843717:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766845.099569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766845.867622:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766846.123603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766846.315582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766846.315592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766846.315597:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766846.315606:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2574 00000100:00080000:3.0:1775766846.315636:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766846.315642:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766846.315681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861433472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766846.315709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766846.315712:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766846.315714:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766846.315717:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2574 00000100:00080000:3.0:1775766846.315719:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766846.315722:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766846.315733:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766846.315767:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861433600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766846.315775:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861433472:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766846.315785:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766846.315791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766846.315793:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766846.315799:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2574 00000100:00080000:3.0:1775766846.315804:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766846.315809:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766846.315815:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766846.315828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861433728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766846.315857:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766846.315901:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766846/real 1775766846] req@ffffa0e9d0540a80 x1862023861433472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766846.315917:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861433472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766846.315927:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766846.315931:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766846.315935:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861433472:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766846.315957:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766846.315973:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766846.315981:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766846/real 1775766846] req@ffffa0e9d0542d80 x1862023861433600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766846.315989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861433600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766846.315995:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766846.315997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766846.315999:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433600:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766846.316003:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766846.316013:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766846.316018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766846/real 1775766846] req@ffffa0e9d0541f80 x1862023861433728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766846.316034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861433728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766846.316039:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766846.316041:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766846.316043:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433728:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766846.891641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766847.147632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766847.915640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766848.171568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766848.939842:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766849.195625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766849.963693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766850.219562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766850.987769:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766851.243616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766851.435577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766851.435606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766851.435611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766851.435625:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2579 00000100:00080000:3.0:1775766851.435634:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766851.435639:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766851.435680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861433856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766851.435748:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766851.435763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766851.435765:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766851.435769:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2579 00000100:00080000:3.0:1775766851.435774:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766851.435778:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766851.435789:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766851.435804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861433984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766851.435815:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861433856:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766851.435815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766851.435823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766851.435826:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766851.435831:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2579 00000100:00080000:3.0:1775766851.435855:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766851.435859:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766851.435865:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766851.435877:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861434112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766851.435896:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766851.435940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766851/real 1775766851] req@ffffa0e9d0541180 x1862023861433856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766851.435960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861433856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766851.435972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766851.435976:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766851.435981:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861433856:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766851.436005:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766851.436029:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766851.436042:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766851/real 1775766851] req@ffffa0e9d0543480 x1862023861433984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766851.436054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861433984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766851.436076:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766851.436080:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766851.436083:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861433984:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766851.436090:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766851.436112:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766851.436124:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766851/real 1775766851] req@ffffa0e9d0543100 x1862023861434112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766851.436147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861434112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766851.436155:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766851.436158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766851.436161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434112:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766852.011713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766852.267564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766853.035678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766853.291626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766854.059644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766854.315670:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766855.083792:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766855.339611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766856.107735:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766856.363591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766856.555613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766856.555622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766856.555627:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766856.555666:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2585 00000100:00080000:1.0:1775766856.555676:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766856.555682:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766856.555737:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861434240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766856.555770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766856.555775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766856.555777:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766856.555799:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2585 00000100:00080000:1.0:1775766856.555805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766856.555810:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766856.555822:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766856.555842:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861434368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766856.555872:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861434240:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766856.555892:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766856.555895:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766856.555896:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766856.555901:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2585 00000100:00080000:1.0:1775766856.555905:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766856.555908:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766856.555913:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766856.555922:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861434496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766856.555942:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766856.556006:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766856/real 1775766856] req@ffffa0e9c8deea00 x1862023861434240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766856.556027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861434240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766856.556039:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766856.556043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766856.556055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861434240:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766856.556080:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766856.556124:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766856.556138:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766856/real 1775766856] req@ffffa0e9c8dee680 x1862023861434368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766856.556154:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861434368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766856.556181:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766856.556186:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766856.556191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434368:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766856.556200:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766856.556228:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766856.556242:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766856/real 1775766856] req@ffffa0e9c8ded880 x1862023861434496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766856.556259:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861434496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766856.556270:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766856.556275:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766856.556279:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434496:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766857.131708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766857.387635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766858.155690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766858.411605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766859.179705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766859.435590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766860.203783:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766860.459560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766861.227666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766861.483630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766861.675653:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766861.675666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766861.675691:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766861.675704:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2590 00000100:00080000:0.0:1775766861.675718:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766861.675727:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766861.675773:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861434624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766861.675810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766861.675815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766861.675818:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766861.675824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2590 00000100:00080000:0.0:1775766861.675828:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766861.675834:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766861.675842:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861434624:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775766861.675846:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766861.675864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861434752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766861.675878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766861.675882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766861.675885:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766861.675889:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2590 00000100:00080000:0.0:1775766861.675904:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766861.675909:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766861.675917:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766861.675917:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766861.675935:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861434880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766861.675974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766861/real 1775766861] req@ffffa0e9d09dd880 x1862023861434624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766861.675996:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861434624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766861.676007:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766861.676011:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766861.676016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861434624:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766861.676045:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766861.676069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766861.676084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766861/real 1775766861] req@ffffa0e9d09df800 x1862023861434752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766861.676105:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861434752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766861.676111:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766861.676114:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766861.676116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434752:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766861.676121:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766861.676138:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766861.676146:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766861/real 1775766861] req@ffffa0e9d09dc700 x1862023861434880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766861.676154:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861434880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766861.676159:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766861.676161:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766861.676163:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861434880:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766862.251709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766862.507615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766863.275771:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766863.531601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766864.299730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766864.555636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766865.323649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766865.579569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766866.347678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766866.603605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766866.795692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766866.795702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766866.795706:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766866.795715:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2595 00000100:00080000:3.0:1775766866.795725:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766866.795730:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766866.795768:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861435008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766866.795794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766866.795797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766866.795798:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766866.795801:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2595 00000100:00080000:3.0:1775766866.795803:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766866.795805:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766866.795813:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766866.795822:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861435136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766866.795830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766866.795833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766866.795834:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766866.795837:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2595 00000100:00080000:3.0:1775766866.795839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766866.795843:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861435008:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766866.795843:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766866.795866:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766866.795877:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861435264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766866.795900:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766866.795937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766866/real 1775766866] req@ffffa0e9d0543100 x1862023861435008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766866.795950:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861435008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766866.795976:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766866.795979:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766866.795982:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861435008:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766866.795988:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766866.796005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766866.796014:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766866/real 1775766866] req@ffffa0e9d0543480 x1862023861435136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766866.796022:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861435136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766866.796027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766866.796030:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766866.796032:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435136:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766866.796049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766866.796060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766866.796065:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766866/real 1775766866] req@ffffa0e9d0541180 x1862023861435264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766866.796073:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861435264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766866.796078:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766866.796080:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766866.796082:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435264:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766867.371628:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766867.627581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766868.395733:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766868.651618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766869.419668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766869.675586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766870.443625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766870.699610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766871.467677:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766871.723597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766871.915598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766871.915608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766871.915614:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766871.915624:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2600 00000100:00080000:3.0:1775766871.915635:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766871.915642:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766871.915683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861435392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766871.915711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766871.915715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766871.915717:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766871.915721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2600 00000100:00080000:3.0:1775766871.915725:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766871.915729:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766871.915741:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766871.915754:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861435520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766871.915764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766871.915765:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861435392:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766871.915768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766871.915770:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766871.915773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2600 00000100:00080000:3.0:1775766871.915793:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766871.915797:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766871.915802:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766871.915812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861435648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766871.915838:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766871.915892:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766871/real 1775766871] req@ffffa0e9d0541f80 x1862023861435392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766871.915909:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861435392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766871.915917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766871.915920:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766871.915923:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861435392:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766871.915947:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766871.915965:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766871.915974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766871/real 1775766871] req@ffffa0e9d0542d80 x1862023861435520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766871.915981:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861435520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766871.915987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766871.915989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766871.915991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435520:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766871.915994:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766871.916004:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766871.916010:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766871/real 1775766871] req@ffffa0e9d0540a80 x1862023861435648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766871.916024:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861435648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766871.916030:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766871.916032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766871.916034:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435648:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766872.491687:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766872.747602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766873.515650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766873.771585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766874.539657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766874.795601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766875.563669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766875.819635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766876.587654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766876.843608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766877.035582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766877.035591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766877.035596:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766877.035606:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2605 00000100:00080000:3.0:1775766877.035618:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766877.035626:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766877.035674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861435776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766877.035720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766877.035723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766877.035725:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766877.035728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2605 00000100:00080000:3.0:1775766877.035732:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766877.035735:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766877.035745:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766877.035756:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861435904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766877.035764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766877.035781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766877.035783:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766877.035787:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2605 00000100:00080000:3.0:1775766877.035790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766877.035794:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861435776:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766877.035797:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766877.035806:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766877.035816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861436032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766877.035881:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766877.035960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766877/real 1775766877] req@ffffa0e9d0540e00 x1862023861435776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766877.035996:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861435776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766877.036006:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766877.036010:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766877.036013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861435776:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766877.036019:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766877.036040:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766877.036049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766877/real 1775766877] req@ffffa0e9d0540700 x1862023861435904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766877.036078:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861435904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766877.036094:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766877.036097:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766877.036099:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861435904:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766877.036122:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766877.036152:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766877.036160:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766877/real 1775766877] req@ffffa0e9d0543800 x1862023861436032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766877.036167:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861436032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766877.036173:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766877.036175:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766877.036177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436032:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766877.611620:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766877.867570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766878.635768:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766878.891571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766879.659647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766879.915580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766880.683624:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766880.939571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766881.707665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766881.963554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766882.155622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766882.155631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766882.155635:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766882.155644:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2610 00000100:00080000:1.0:1775766882.155653:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766882.155659:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766882.155698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861436160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766882.155725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766882.155746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766882.155748:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766882.155752:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2610 00000100:00080000:1.0:1775766882.155755:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766882.155759:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766882.155767:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766882.155779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861436160:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766882.155779:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861436288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766882.155796:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766882.155798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766882.155800:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766882.155803:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2610 00000100:00080000:1.0:1775766882.155805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766882.155807:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766882.155810:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766882.155818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861436416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766882.155856:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766882.155897:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766882/real 1775766882] req@ffffa0e9c8deca80 x1862023861436160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766882.155914:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861436160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766882.155923:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766882.155927:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766882.155931:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861436160:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766882.155955:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766882.155981:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766882.155992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766882/real 1775766882] req@ffffa0e9c8dedf80 x1862023861436288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766882.156001:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861436288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766882.156007:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766882.156010:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766882.156012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436288:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766882.156017:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766882.156030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766882.156051:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766882/real 1775766882] req@ffffa0e9c8dece00 x1862023861436416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766882.156064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861436416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766882.156072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766882.156076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766882.156080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436416:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766882.731675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766882.987568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766883.755718:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766884.011554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766884.779662:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766885.035595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766885.803632:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766886.059628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766886.827640:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766887.083582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766887.275682:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766887.275695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766887.275701:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766887.275711:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2615 00000100:00080000:3.0:1775766887.275723:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766887.275730:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766887.275783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861436544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766887.275815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766887.275819:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766887.275821:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766887.275825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2615 00000100:00080000:3.0:1775766887.275829:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766887.275832:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766887.275840:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766887.275851:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861436672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766887.275878:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766887.275882:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766887.275883:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766887.275888:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2615 00000100:00080000:3.0:1775766887.275893:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766887.275897:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766887.275906:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766887.275907:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861436544:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766887.275930:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861436800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766887.276001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766887.276045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766887/real 1775766887] req@ffffa0e9d0542300 x1862023861436544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766887.276062:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861436544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766887.276070:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766887.276074:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766887.276078:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861436544:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766887.276084:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766887.276099:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766887.276107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766887/real 1775766887] req@ffffa0e9d0540380 x1862023861436672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766887.276115:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861436672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766887.276137:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766887.276140:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766887.276142:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436672:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766887.276164:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766887.276179:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766887.276186:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766887/real 1775766887] req@ffffa0e9d0541c00 x1862023861436800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766887.276193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861436800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766887.276198:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766887.276200:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766887.276202:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861436800:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766887.851722:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766888.107653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766888.875641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766889.131599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766889.899668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766890.155629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766890.923694:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766891.179569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766891.947688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766892.203615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766892.395637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766892.395646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766892.395651:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766892.395663:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2620 00000100:00080000:3.0:1775766892.395675:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766892.395683:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766892.395738:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861436928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766892.395765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766892.395783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766892.395785:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766892.395788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2620 00000100:00080000:3.0:1775766892.395791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766892.395795:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766892.395817:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861436928:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775766892.395822:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766892.395835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861437056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766892.395844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766892.395847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766892.395849:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766892.395853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2620 00000100:00080000:3.0:1775766892.395857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766892.395860:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766892.395864:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766892.395871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861437184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766892.395965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437056:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766892.395993:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437184:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766892.971721:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766893.227645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766893.995695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766894.251651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766895.019662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766895.275640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775766895.467802:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775766895.467878:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766895.467897:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766895.467948:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775766895.467959:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766895.468043:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766892/real 1775766895] req@ffffa0e9d0540000 x1862023861437056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766895.468079:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861437056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766895.468120:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766895.468126:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766895.468130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437056:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766895.468139:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766892/real 1775766895] req@ffffa0e9d0542a00 x1862023861437184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766895.468152:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861437184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766895.468159:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766895.468163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766895.468166:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437184:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766895.468172:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766892/real 1775766895] req@ffffa0e9d0543b80 x1862023861436928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766895.468183:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861436928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766895.468190:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766895.468192:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766895.468196:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861436928:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775766896.043773:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766896.299620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766897.067715:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766897.323577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766897.515601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766897.515611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766897.515616:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766897.515627:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2626 00000100:00080000:0.0:1775766897.515653:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766897.515661:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766897.515699:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861437312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766897.515731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766897.515734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766897.515735:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766897.515739:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2626 00000100:00080000:0.0:1775766897.515741:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766897.515744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766897.515754:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766897.515763:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861437440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766897.515773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766897.515777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766897.515779:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766897.515785:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2626 00000100:00080000:0.0:1775766897.515790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766897.515792:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861437312:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766897.515796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766897.515806:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766897.515818:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861437568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766897.515894:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766897.515964:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766897/real 1775766897] req@ffffa0e9d09ded80 x1862023861437312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766897.515999:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861437312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766897.516029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766897.516035:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766897.516041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861437312:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766897.516077:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766897.516109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766897.516125:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766897/real 1775766897] req@ffffa0e9d09dfb80 x1862023861437440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766897.516141:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861437440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766897.516152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766897.516156:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766897.516161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437440:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766897.516168:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766897.516195:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766897.516223:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766897/real 1775766897] req@ffffa0e9d09df100 x1862023861437568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766897.516237:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861437568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766897.516247:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766897.516251:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766897.516254:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437568:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766898.091687:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766898.347582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766899.115623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766899.371625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766900.139757:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766900.395627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766901.163730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766901.419603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766902.187699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766902.443593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766902.635612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766902.635621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766902.635627:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766902.635637:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2631 00000100:00080000:1.0:1775766902.635649:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766902.635655:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766902.635697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861437696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766902.635732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766902.635735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766902.635736:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766902.635739:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2631 00000100:00080000:1.0:1775766902.635742:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766902.635745:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766902.635755:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766902.635764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861437824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766902.635774:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861437696:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766902.635774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766902.635777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766902.635778:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766902.635782:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2631 00000100:00080000:1.0:1775766902.635784:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766902.635786:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766902.635798:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766902.635810:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861437952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766902.635869:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766902.635923:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766902/real 1775766902] req@ffffa0e9c8dec700 x1862023861437696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766902.635941:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861437696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766902.635949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766902.635952:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766902.635956:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861437696:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766902.635979:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766902.635995:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766902.636004:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766902/real 1775766902] req@ffffa0e9c8defb80 x1862023861437824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766902.636012:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861437824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766902.636018:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766902.636020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766902.636022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437824:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766902.636026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766902.636036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766902.636041:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766902/real 1775766902] req@ffffa0e9c8def100 x1862023861437952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766902.636059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861437952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766902.636065:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766902.636069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766902.636072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861437952:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766903.211672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766903.467586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766904.235784:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766904.491616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766905.259721:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766905.515639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766906.283664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766906.539579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766907.307688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766907.563577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766907.755566:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766907.755575:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766907.755580:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766907.755590:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2636 00000100:00080000:1.0:1775766907.755602:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766907.755608:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766907.755651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861438080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766907.755678:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766907.755681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766907.755683:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766907.755686:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2636 00000100:00080000:1.0:1775766907.755688:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766907.755691:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766907.755700:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766907.755709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861438208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766907.755717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766907.755726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766907.755728:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766907.755731:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2636 00000100:00100000:0.0:1775766907.755742:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861438080:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766907.755750:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766907.755755:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766907.755760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766907.755769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861438336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766907.755855:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766907.755915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766907/real 1775766907] req@ffffa0e9c8def100 x1862023861438080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766907.755935:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861438080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766907.755945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766907.755949:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766907.755952:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861438080:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766907.755959:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766907.755979:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766907.755989:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766907/real 1775766907] req@ffffa0e9c8defb80 x1862023861438208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766907.756010:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861438208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766907.756017:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766907.756020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766907.756022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438208:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766907.756045:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766907.756060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766907.756067:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766907/real 1775766907] req@ffffa0e9c8dec700 x1862023861438336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766907.756076:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861438336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766907.756082:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766907.756085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766907.756088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438336:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766908.331712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766908.587643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766909.355743:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766909.611601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766910.379628:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766910.635588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766911.403623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766911.659670:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766912.427635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766912.683590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766912.875637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766912.875646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766912.875651:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766912.875662:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2641 00000100:00080000:3.0:1775766912.875674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766912.875681:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766912.875725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861438464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766912.875770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766912.875773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766912.875775:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766912.875778:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2641 00000100:00080000:3.0:1775766912.875782:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766912.875785:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766912.875795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766912.875805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861438592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766912.875813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766912.875816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766912.875818:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766912.875821:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2641 00000100:00080000:3.0:1775766912.875824:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766912.875827:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766912.875832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766912.875842:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861438720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766912.875854:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861438464:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766912.875934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766912.875996:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766912/real 1775766912] req@ffffa0e9d0541500 x1862023861438464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766912.876059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861438464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766912.876076:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766912.876084:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766912.876090:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861438464:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766912.876100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766912.876127:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766912.876139:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766912/real 1775766912] req@ffffa0e9d0541880 x1862023861438592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766912.876151:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861438592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766912.876158:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766912.876162:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766912.876164:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438592:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766912.876190:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766912.876205:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766912.876213:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766912/real 1775766912] req@ffffa0e9d0542680 x1862023861438720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766912.876223:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861438720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766912.876242:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766912.876245:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766912.876248:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438720:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766913.451684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766913.707592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766914.475789:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766914.731571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766915.499666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766915.755660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766916.523660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766916.779589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766917.547641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766917.803631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766917.995640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766917.995650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766917.995654:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766917.995663:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2646 00000100:00080000:0.0:1775766917.995672:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766917.995678:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766917.995715:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861438848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766917.995744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766917.995746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766917.995748:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766917.995750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2646 00000100:00080000:0.0:1775766917.995753:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766917.995755:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766917.995763:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766917.995770:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861438976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766917.995797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766917.995800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766917.995802:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766917.995804:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2646 00000100:00080000:0.0:1775766917.995807:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766917.995811:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766917.995816:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766917.995823:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861438848:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766917.995830:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861439104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766917.995927:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766917.995978:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766917/real 1775766917] req@ffffa0e9d09dce00 x1862023861438848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766917.996019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861438848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766917.996033:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766917.996038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766917.996043:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861438848:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766917.996053:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766917.996083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766917.996097:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766917/real 1775766917] req@ffffa0e9d09dc380 x1862023861438976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766917.996111:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861438976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766917.996121:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766917.996125:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766917.996145:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861438976:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766917.996174:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766917.996198:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766917.996210:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766917/real 1775766917] req@ffffa0e9d09dc000 x1862023861439104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766917.996224:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861439104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766917.996234:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766917.996238:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766917.996242:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439104:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766918.571643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766918.827655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766919.595722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766919.851555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766920.619707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766920.875573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766921.643735:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766921.899574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766922.667752:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766922.923595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766923.115696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766923.115707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766923.115713:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766923.115725:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2651 00000100:00080000:1.0:1775766923.115740:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766923.115748:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766923.115799:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861439232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766923.115837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766923.115841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766923.115843:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766923.115847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2651 00000100:00080000:1.0:1775766923.115850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766923.115875:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766923.115888:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766923.115894:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861439232:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766923.115907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861439360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766923.115920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766923.115924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766923.115927:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766923.115931:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2651 00000100:00080000:1.0:1775766923.115955:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766923.115958:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766923.115964:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766923.115995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861439488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766923.116044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766923.116095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766923/real 1775766923] req@ffffa0e9c8dece00 x1862023861439232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766923.116135:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861439232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766923.116148:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766923.116152:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766923.116157:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861439232:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766923.116218:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766923.116253:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766923.116276:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766923/real 1775766923] req@ffffa0e9c8dedf80 x1862023861439360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766923.116302:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861439360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766923.116334:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766923.116339:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766923.116344:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439360:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766923.116354:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766923.116384:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766923.116396:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766923/real 1775766923] req@ffffa0e9c8deca80 x1862023861439488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766923.116408:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861439488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766923.116416:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766923.116419:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766923.116422:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439488:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766923.691654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766923.947562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766924.715661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766924.971610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766925.739657:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766925.995582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766926.763648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766927.019612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766927.787671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766928.043641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766928.235602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766928.235615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766928.235620:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766928.235632:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2656 00000100:00080000:3.0:1775766928.235673:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766928.235679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766928.235713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861439616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766928.235736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766928.235738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766928.235740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766928.235747:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2656 00000100:00080000:3.0:1775766928.235750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766928.235752:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766928.235763:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766928.235772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861439744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766928.235779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766928.235781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766928.235782:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766928.235785:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2656 00000100:00080000:3.0:1775766928.235786:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766928.235788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766928.235791:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766928.235798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861439872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766928.235849:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861439616:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766928.235968:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766928.236030:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766928/real 1775766928] req@ffffa0e9d0542680 x1862023861439616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766928.236055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861439616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766928.236067:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766928.236072:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766928.236077:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861439616:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766928.236086:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766928.236107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766928.236117:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766928/real 1775766928] req@ffffa0e9d0541880 x1862023861439744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766928.236126:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861439744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766928.236132:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766928.236135:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766928.236137:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439744:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766928.236141:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766928.236152:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766928.236158:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766928/real 1775766928] req@ffffa0e9d0541500 x1862023861439872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766928.236179:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861439872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766928.236193:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766928.236195:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766928.236198:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861439872:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766928.811710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766929.067670:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766929.835680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766930.091620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766930.859734:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766931.115602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766931.883694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766932.139654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766932.907653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766933.163667:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766933.355646:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766933.355660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766933.355665:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766933.355677:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2661 00000100:00080000:1.0:1775766933.355705:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766933.355713:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766933.355771:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861440000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766933.355809:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766933.355813:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766933.355885:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766933.355892:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2661 00000100:00080000:1.0:1775766933.355899:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766933.355904:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766933.355916:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766933.355937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861440128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766933.355940:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861440000:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766933.355971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766933.355974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766933.355976:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766933.355983:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2661 00000100:00080000:1.0:1775766933.355989:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766933.355994:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766933.356001:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766933.356014:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861440256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766933.356086:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766933.356135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766933/real 1775766933] req@ffffa0e9c8ded880 x1862023861440000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766933.356158:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861440000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766933.356172:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766933.356178:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766933.356184:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861440000:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766933.356219:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766933.356247:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766933.356260:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766933/real 1775766933] req@ffffa0e9c8dee680 x1862023861440128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766933.356285:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861440128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766933.356293:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766933.356296:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766933.356299:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766933.356304:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766933.356321:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766933.356329:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766933/real 1775766933] req@ffffa0e9c8deea00 x1862023861440256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766933.356339:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861440256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766933.356345:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766933.356348:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766933.356350:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440256:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766933.931698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766934.187640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766934.955702:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766935.211604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766935.979664:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766936.235610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766937.003684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766937.259608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766938.027676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766938.283595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766938.475568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766938.475577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766938.475582:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766938.475591:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2666 00000100:00080000:1.0:1775766938.475604:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766938.475611:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766938.475654:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861440384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766938.475699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766938.475702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766938.475704:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766938.475707:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2666 00000100:00080000:1.0:1775766938.475710:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766938.475713:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766938.475722:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766938.475733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861440512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766938.475741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766938.475744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766938.475745:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766938.475747:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2666 00000100:00080000:1.0:1775766938.475749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766938.475752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766938.475755:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766938.475764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861440640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766938.475765:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861440384:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766938.475867:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766938.475907:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766938/real 1775766938] req@ffffa0e9c8dedc00 x1862023861440384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766938.475931:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861440384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766938.475940:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766938.475943:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766938.475946:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861440384:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766938.475952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766938.475969:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766938.475988:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766938/real 1775766938] req@ffffa0e9c8def480 x1862023861440512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766938.475995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861440512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766938.476001:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766938.476003:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766938.476005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440512:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766938.476026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766938.476037:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766938.476043:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766938/real 1775766938] req@ffffa0e9c8def800 x1862023861440640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766938.476050:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861440640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766938.476064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766938.476067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766938.476069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440640:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766939.051656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766939.307605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766940.075678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766940.331612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766941.099710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766941.355636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766942.123776:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766942.379601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766943.147719:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766943.403599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766943.595657:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766943.595687:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766943.595691:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766943.595701:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2672 00000100:00080000:3.0:1775766943.595713:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766943.595735:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766943.595774:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861440768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766943.595800:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766943.595808:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766943.595810:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766943.595813:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2672 00000100:00080000:3.0:1775766943.595816:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766943.595822:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766943.595832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766943.595841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861440896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766943.595849:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766943.595854:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861440768:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766943.595854:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766943.595857:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766943.595865:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2672 00000100:00080000:3.0:1775766943.595867:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766943.595869:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766943.595872:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766943.595881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861441024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766943.595937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766943.595981:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766943/real 1775766943] req@ffffa0e9d0543b80 x1862023861440768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766943.596000:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861440768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766943.596013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766943.596018:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766943.596022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861440768:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766943.596050:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766943.596080:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766943.596095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766943/real 1775766943] req@ffffa0e9d0542a00 x1862023861440896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766943.596108:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861440896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766943.596123:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766943.596126:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766943.596130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861440896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766943.596135:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766943.596154:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766943.596162:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766943/real 1775766943] req@ffffa0e9d0540000 x1862023861441024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766943.596169:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861441024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766943.596175:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766943.596177:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766943.596179:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441024:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766944.171773:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766944.427606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766945.195832:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766945.451598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766946.219819:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766946.475716:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766947.243696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766947.499612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766948.267801:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766948.523708:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766948.715611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766948.715623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766948.715628:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766948.715638:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2677 00000100:00080000:2.0:1775766948.715649:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766948.715656:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766948.715732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861441152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766948.715767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766948.715771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766948.715772:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766948.715776:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2677 00000100:00080000:2.0:1775766948.715780:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766948.715784:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766948.715794:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766948.715808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861441280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766948.715843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766948.715849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775766948.715851:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861441152:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775766948.715852:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766948.715862:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2677 00000100:00080000:2.0:1775766948.715865:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766948.715869:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766948.715873:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766948.715907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861441408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766948.715924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766948.715980:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766948/real 1775766948] req@ffffa0e9d0711500 x1862023861441152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766948.715998:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861441152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766948.716018:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766948.716022:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766948.716028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861441152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766948.716039:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766948.716061:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766948.716072:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766948/real 1775766948] req@ffffa0e9d0713480 x1862023861441280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766948.716083:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861441280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766948.716101:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766948.716104:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766948.716106:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441280:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766948.716133:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766948.716149:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766948.716157:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766948/real 1775766948] req@ffffa0e9d0712a00 x1862023861441408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766948.716167:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861441408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766948.716174:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766948.716176:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766948.716179:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441408:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766949.291708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766949.547595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766950.315681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766950.571607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766951.339712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766951.595633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766952.363767:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766952.619610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766953.387680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766953.643645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775766953.835591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766953.835602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766953.835608:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766953.835631:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2682 00000100:00080000:1.0:1775766953.835660:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766953.835668:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775766953.835710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861441536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766953.835741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766953.835745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766953.835748:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766953.835752:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2682 00000100:00080000:1.0:1775766953.835755:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766953.835759:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766953.835770:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766953.835782:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861441664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766953.835796:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766953.835799:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766953.835801:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766953.835807:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2682 00000100:00100000:0.0:1775766953.835808:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861441536:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775766953.835810:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766953.835817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775766953.835823:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766953.835834:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861441792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766953.835895:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766953.835950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766953/real 1775766953] req@ffffa0e9c8dec380 x1862023861441536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766953.835971:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861441536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766953.835983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766953.835987:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766953.835991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861441536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766953.836019:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766953.836043:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766953.836056:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766953/real 1775766953] req@ffffa0e9c8ded500 x1862023861441664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766953.836069:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861441664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766953.836078:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766953.836082:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766953.836086:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766953.836103:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766953.836127:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766953.836139:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766953/real 1775766953] req@ffffa0e9c8deed80 x1862023861441792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766953.836153:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861441792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766953.836162:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766953.836166:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766953.836169:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861441792:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766954.411781:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766954.667640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766955.435712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766955.691588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766956.459676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766956.715673:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766957.483687:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766957.739616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766958.507780:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766958.763639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766958.955719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766958.955731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766958.955735:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766958.955745:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2687 00000100:00080000:3.0:1775766958.955756:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766958.955762:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766958.955804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861441920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766958.955834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766958.955839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766958.955841:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766958.955846:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2687 00000100:00080000:3.0:1775766958.955850:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766958.955856:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766958.955866:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766958.955884:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861442048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766958.955897:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766958.955902:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861441920:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766958.955902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766958.955904:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766958.955908:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2687 00000100:00080000:3.0:1775766958.955912:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766958.955916:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766958.955941:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766958.955954:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861442176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766958.956019:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766958.956055:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442176:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766959.531700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766959.787633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766960.555689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766960.811652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766961.579747:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766961.835604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775766962.027837:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775766962.027894:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766962.027913:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766962.027968:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775766962.027979:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775766962.028038:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766958/real 1775766962] req@ffffa0e9d0540380 x1862023861442048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766962.028069:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861442048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766962.028111:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766962.028117:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766962.028122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766962.028132:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766958/real 1775766962] req@ffffa0e9d0542300 x1862023861442176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766962.028147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861442176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766962.028156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766962.028187:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766962.028191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766962.028199:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766958/real 1775766962] req@ffffa0e9d0541c00 x1862023861441920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766962.028214:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861441920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766962.028222:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766962.028226:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766962.028229:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861441920:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775766962.603668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766962.859587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766963.627687:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766963.883614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766964.075669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766964.075681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766964.075687:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766964.075701:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2692 00000100:00080000:2.0:1775766964.075725:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766964.075733:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766964.075792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861442304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766964.075827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766964.075832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766964.075834:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766964.075838:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2692 00000100:00080000:2.0:1775766964.075842:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766964.075846:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766964.075858:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766964.075871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861442432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766964.075882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766964.075885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766964.075887:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766964.075891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2692 00000100:00080000:2.0:1775766964.075895:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766964.075898:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766964.075904:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766964.075915:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861442560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766964.075977:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861442304:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775766964.076083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766964.076171:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766964/real 1775766964] req@ffffa0e9d0710e00 x1862023861442304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766964.076194:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861442304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766964.076207:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766964.076211:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766964.076217:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861442304:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766964.076225:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766964.076251:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766964.076269:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766964/real 1775766964] req@ffffa0e9d0710a80 x1862023861442432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766964.076284:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861442432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766964.076295:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766964.076299:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766964.076302:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442432:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766964.076308:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766964.076335:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766964.076347:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766964/real 1775766964] req@ffffa0e9d0712680 x1862023861442560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766964.076361:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861442560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766964.076384:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766964.076388:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766964.076392:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442560:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766964.651732:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766964.907650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766965.675736:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766965.931657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766966.699665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766966.955586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766967.723695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766967.979593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766968.747644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766969.003564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766969.195596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766969.195605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766969.195610:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766969.195620:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2697 00000100:00080000:3.0:1775766969.195632:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766969.195639:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766969.195681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861442688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766969.195708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766969.195711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766969.195712:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766969.195716:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2697 00000100:00080000:3.0:1775766969.195718:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766969.195720:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766969.195730:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766969.195739:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861442816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766969.195747:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766969.195750:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766969.195751:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766969.195754:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2697 00000100:00100000:0.0:1775766969.195768:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861442688:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766969.195773:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766969.195779:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766969.195783:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766969.195802:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861442944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766969.195854:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766969.195896:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766969/real 1775766969] req@ffffa0e9d0543800 x1862023861442688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766969.195912:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861442688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766969.195921:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766969.195924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766969.195927:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861442688:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775766969.195933:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766969.195948:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766969.195956:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766969/real 1775766969] req@ffffa0e9d0540700 x1862023861442816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766969.195964:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861442816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766969.195976:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766969.195979:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766969.195981:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442816:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775766969.196004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775766969.196019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775766969.196025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766969/real 1775766969] req@ffffa0e9d0540e00 x1862023861442944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766969.196033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861442944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766969.196038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766969.196040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775766969.196041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861442944:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766969.771817:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766970.027615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766970.795656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766971.051748:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766971.819656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766972.075596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766972.843661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766973.099630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766973.867691:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766974.123602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766974.315624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766974.315632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766974.315637:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766974.315648:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2702 00000100:00080000:0.0:1775766974.315661:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766974.315670:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766974.315727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861443072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766974.315758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766974.315761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766974.315763:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766974.315766:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2702 00000100:00080000:0.0:1775766974.315769:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766974.315772:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766974.315782:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766974.315792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861443200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766974.315803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766974.315807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766974.315811:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766974.315819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2702 00000100:00100000:1.0:1775766974.315822:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861443072:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766974.315825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766974.315831:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766974.315838:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766974.315861:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861443328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766974.315911:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766974.315949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766974/real 1775766974] req@ffffa0e9d09de300 x1862023861443072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766974.315964:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861443072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766974.315973:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766974.315977:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766974.315980:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861443072:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766974.316000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766974.316015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766974.316022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766974/real 1775766974] req@ffffa0e9d09dea00 x1862023861443200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766974.316030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861443200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766974.316036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766974.316038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766974.316041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766974.316047:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766974.316067:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766974.316077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766974/real 1775766974] req@ffffa0e9d09df480 x1862023861443328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766974.316105:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861443328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766974.316112:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766974.316114:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766974.316116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443328:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766974.891667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766975.147578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766975.915667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766976.171606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766976.939631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766977.195547:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766977.963619:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766978.219582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766978.987709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766979.243555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766979.435635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766979.435644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766979.435648:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766979.435658:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2707 00000100:00080000:0.0:1775766979.435667:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766979.435673:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766979.435714:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861443456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766979.435738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766979.435742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766979.435744:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766979.435748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2707 00000100:00080000:0.0:1775766979.435752:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766979.435756:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766979.435765:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766979.435778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861443584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766979.435789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766979.435792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775766979.435792:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861443456:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766979.435793:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766979.435797:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2707 00000100:00080000:0.0:1775766979.435800:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766979.435804:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766979.435827:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766979.435837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861443712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766979.435853:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766979.435894:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766979/real 1775766979] req@ffffa0e9d09df480 x1862023861443456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766979.435913:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861443456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766979.435921:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766979.435924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766979.435927:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861443456:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766979.435946:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766979.435963:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766979.435972:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766979/real 1775766979] req@ffffa0e9d09dea00 x1862023861443584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766979.435979:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861443584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766979.435995:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766979.435997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766979.435999:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443584:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766979.436003:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766979.436016:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766979.436023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766979/real 1775766979] req@ffffa0e9d09de300 x1862023861443712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766979.436030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861443712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766979.436035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766979.436036:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766979.436038:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443712:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766980.011629:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766980.267703:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766981.035670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766981.291614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766982.059664:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766982.315590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766983.083708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766983.339591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766984.107688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766984.363579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775766984.555585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766984.555595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766984.555600:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766984.555611:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2713 00000100:00080000:0.0:1775766984.555623:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766984.555630:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775766984.555674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861443840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766984.555704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766984.555708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766984.555709:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766984.555713:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2713 00000100:00080000:0.0:1775766984.555716:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766984.555719:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766984.555730:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766984.555739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861443968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766984.555769:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861443840:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775766984.555772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766984.555778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766984.555780:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766984.555785:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2713 00000100:00080000:0.0:1775766984.555789:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766984.555792:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775766984.555806:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766984.555817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861444096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766984.555859:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766984.555914:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766984/real 1775766984] req@ffffa0e9d09dc000 x1862023861443840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766984.555961:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861443840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766984.555972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766984.555975:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766984.555986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861443840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766984.556011:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766984.556034:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766984.556054:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766984/real 1775766984] req@ffffa0e9d09dc380 x1862023861443968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766984.556065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861443968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766984.556072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766984.556074:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766984.556077:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861443968:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766984.556082:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766984.556096:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766984.556103:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766984/real 1775766984] req@ffffa0e9d09dce00 x1862023861444096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766984.556120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861444096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766984.556126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766984.556128:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766984.556130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444096:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766985.131654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766985.387579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766986.155684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766986.411657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766987.179761:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766987.435625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766988.203666:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766988.459584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766989.227649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766989.483690:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775766989.675630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766989.675640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766989.675646:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766989.675656:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2718 00000100:00080000:2.0:1775766989.675670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766989.675676:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775766989.675719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861444224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766989.675745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766989.675762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766989.675764:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766989.675768:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2718 00000100:00080000:2.0:1775766989.675772:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766989.675776:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766989.675786:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766989.675793:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861444224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775766989.675800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861444352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766989.675810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766989.675812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766989.675813:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766989.675816:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2718 00000100:00080000:2.0:1775766989.675818:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766989.675820:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775766989.675823:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766989.675832:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861444480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766989.675867:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766989.675915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766989/real 1775766989] req@ffffa0e9d0712d80 x1862023861444224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766989.675931:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861444224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766989.675939:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766989.675943:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766989.675946:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861444224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766989.675969:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766989.675985:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766989.675994:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766989/real 1775766989] req@ffffa0e9d0710700 x1862023861444352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766989.676002:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861444352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766989.676008:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766989.676010:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766989.676012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444352:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766989.676016:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766989.676026:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766989.676031:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766989/real 1775766989] req@ffffa0e9d0711f80 x1862023861444480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766989.676039:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861444480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766989.676053:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766989.676056:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766989.676058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444480:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766990.251665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766990.507632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766991.275657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766991.531597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766992.299902:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766992.555650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766993.323661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766993.579600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766994.347717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766994.603612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766994.795675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766994.795684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766994.795690:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766994.795701:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2723 00000100:00080000:3.0:1775766994.795714:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766994.795723:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766994.795774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861444608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766994.795821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766994.795826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766994.795829:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766994.795833:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2723 00000100:00080000:3.0:1775766994.795839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766994.795844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766994.795857:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766994.795876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861444736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766994.795886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766994.795933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775766994.795934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861444608:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775766994.795936:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766994.795946:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2723 00000100:00080000:3.0:1775766994.795950:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766994.795954:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766994.795959:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766994.795972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861444864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766994.796031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766994.796077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766994/real 1775766994] req@ffffa0e9d0540a80 x1862023861444608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766994.796115:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861444608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766994.796125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766994.796129:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766994.796133:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861444608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766994.796140:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766994.796158:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766994.796167:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766994/real 1775766994] req@ffffa0e9d0542d80 x1862023861444736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766994.796176:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861444736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766994.796210:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766994.796213:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766994.796216:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444736:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766994.796273:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766994.796299:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766994.796314:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766994/real 1775766994] req@ffffa0e9d0541f80 x1862023861444864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766994.796329:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861444864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766994.796339:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766994.796343:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766994.796347:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861444864:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775766995.371698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766995.627591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766996.395742:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766996.651602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766997.419675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766997.675591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766998.443661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766998.699553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775766999.467642:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766999.723593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775766999.915583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766999.915594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766999.915598:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766999.915608:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2728 00000100:00080000:3.0:1775766999.915617:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766999.915623:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775766999.915661:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861444992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766999.915688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766999.915691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766999.915694:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766999.915698:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2728 00000100:00080000:3.0:1775766999.915702:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766999.915707:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766999.915720:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766999.915730:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861444992:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775766999.915736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861445120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766999.915782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766999.915785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766999.915787:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766999.915792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2728 00000100:00080000:3.0:1775766999.915798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766999.915801:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:3.0:1775766999.915807:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775766999.915814:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766999.915829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861445248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766999.915845:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766999/real 1775766999] req@ffffa0e9d0541180 x1862023861444992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766999.915860:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861444992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766999.915868:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766999.915871:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766999.915874:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861444992:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775766999.915903:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766999.915917:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766999.915924:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766999/real 1775766999] req@ffffa0e9d0543480 x1862023861445120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766999.915939:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861445120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766999.915955:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766999.915958:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766999.915960:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445120:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775766999.915963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775766999.915976:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775766999.915982:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766999/real 1775766999] req@ffffa0e9d0543100 x1862023861445248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766999.915989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861445248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766999.915994:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766999.915996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775766999.915999:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445248:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767000.491676:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767000.747613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767001.515657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767001.771568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767002.539729:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767002.795636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767003.563718:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767003.819620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767004.587775:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767004.843622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767005.035599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767005.035609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767005.035613:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767005.035622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2733 00000100:00080000:3.0:1775767005.035632:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767005.035638:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767005.035677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861445376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767005.035702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767005.035704:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767005.035706:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767005.035708:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2733 00000100:00080000:3.0:1775767005.035711:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767005.035713:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767005.035738:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767005.035744:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861445376:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767005.035750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861445504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767005.035759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767005.035761:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767005.035763:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767005.035765:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2733 00000100:00080000:3.0:1775767005.035768:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767005.035770:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767005.035773:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767005.035781:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861445632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767005.035805:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767005.035842:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767005/real 1775767005] req@ffffa0e9d0543100 x1862023861445376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767005.035856:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861445376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767005.035874:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767005.035877:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767005.035880:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861445376:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767005.035899:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767005.035916:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767005.035925:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767005/real 1775767005] req@ffffa0e9d0543480 x1862023861445504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767005.035933:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861445504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767005.035938:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767005.035940:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767005.035942:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445504:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767005.035946:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767005.035956:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767005.035962:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767005/real 1775767005] req@ffffa0e9d0541180 x1862023861445632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767005.035969:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861445632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767005.035974:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767005.035976:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767005.035977:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445632:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767005.611686:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767005.867612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767006.635688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767006.891654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767007.659729:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767007.915575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767008.683678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767008.939634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767009.707750:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767009.963676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767010.155652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767010.155668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767010.155677:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767010.155693:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2738 00000100:00080000:2.0:1775767010.155712:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767010.155724:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767010.155785:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861445760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767010.155825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767010.155830:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767010.155832:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767010.155837:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2738 00000100:00080000:2.0:1775767010.155842:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767010.155847:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767010.155860:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767010.155874:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861445888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767010.155888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767010.155914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775767010.155918:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861445760:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767010.155918:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767010.155929:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2738 00000100:00080000:2.0:1775767010.155937:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767010.155942:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767010.155949:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767010.155962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861446016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767010.156060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767010.156129:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767010/real 1775767010] req@ffffa0e9d0711c00 x1862023861445760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767010.156158:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861445760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767010.156177:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767010.156185:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767010.156192:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861445760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767010.156205:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767010.156256:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767010.156278:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767010/real 1775767010] req@ffffa0e9d0713b80 x1862023861445888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767010.156294:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861445888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767010.156304:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767010.156307:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767010.156311:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861445888:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767010.156372:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767010.156400:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767010.156412:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767010/real 1775767010] req@ffffa0e9d0710000 x1862023861446016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767010.156441:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861446016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767010.156451:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767010.156455:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767010.156459:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446016:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767010.731700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767010.987637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767011.755654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767012.011589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767012.779660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767013.035570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767013.803701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767014.059606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767014.827639:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767015.083586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767015.275565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767015.275575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767015.275580:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767015.275590:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2743 00000100:00080000:2.0:1775767015.275601:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767015.275607:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767015.275658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861446144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767015.275685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767015.275688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767015.275689:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767015.275693:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2743 00000100:00080000:2.0:1775767015.275695:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767015.275720:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767015.275726:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861446144:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775767015.275730:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767015.275742:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861446272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767015.275751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767015.275754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767015.275756:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767015.275759:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2743 00000100:00080000:2.0:1775767015.275763:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767015.275765:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767015.275769:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767015.275779:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861446400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767015.275799:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767015.275847:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767015/real 1775767015] req@ffffa0e9d0710380 x1862023861446144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767015.275864:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861446144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767015.275884:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767015.275888:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767015.275891:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861446144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767015.275921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767015.275940:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767015.275950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767015/real 1775767015] req@ffffa0e9d0711180 x1862023861446272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767015.275960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861446272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767015.275995:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767015.276028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767015.276033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767015.276042:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767015.276081:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767015.276098:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767015/real 1775767015] req@ffffa0e9d0710000 x1862023861446400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767015.276112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861446400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767015.276119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767015.276122:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767015.276124:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446400:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767015.851757:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767016.107651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767016.875669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767017.131627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767017.899672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767018.155625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767018.923755:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767019.179627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767019.947817:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767020.203656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767020.395646:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767020.395657:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767020.395663:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767020.395676:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2748 00000100:00080000:1.0:1775767020.395695:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767020.395705:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767020.395762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861446528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767020.395806:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767020.395830:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767020.395833:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767020.395841:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2748 00000100:00080000:1.0:1775767020.395849:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767020.395856:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767020.395873:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767020.395881:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861446528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767020.395898:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861446656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767020.395935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767020.395941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767020.395943:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767020.395950:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2748 00000100:00080000:1.0:1775767020.395978:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767020.395984:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775767020.395992:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775767020.395995:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767020.396019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861446784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767020.396049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767020/real 1775767020] req@ffffa0e9c8dee300 x1862023861446528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767020.396072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861446528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767020.396104:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767020.396111:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767020.396117:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861446528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767020.396187:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767020.396221:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767020.396236:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767020/real 1775767020] req@ffffa0e9c8dec000 x1862023861446656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767020.396249:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861446656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767020.396257:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767020.396260:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767020.396263:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446656:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767020.396269:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767020.396285:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767020.396293:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767020/real 1775767020] req@ffffa0e9c8ded180 x1862023861446784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767020.396321:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861446784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767020.396328:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767020.396330:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767020.396333:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861446784:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767020.971770:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767021.227572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767021.995697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767022.251634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767023.019682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767023.275588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767024.043685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767024.299600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767025.067671:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767025.323609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767025.515701:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767025.515716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767025.515721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767025.515735:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2754 00000100:00080000:3.0:1775767025.515749:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767025.515756:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767025.515800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861446912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767025.515891:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767025.515897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767025.515900:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767025.515907:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2754 00000100:00080000:3.0:1775767025.515936:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767025.515938:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861446912:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767025.515944:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767025.515956:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767025.515973:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861447040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767025.515985:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767025.515988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767025.515990:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767025.515994:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2754 00000100:00080000:3.0:1775767025.515998:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767025.516001:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767025.516005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767025.516015:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861447168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767025.516082:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767025.516115:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447168:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767026.091693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767026.347635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767027.115685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767027.371625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767028.139695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767028.395629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:1.0:1775767028.587889:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:1.0:1775767028.587944:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767028.587961:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775767028.588002:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775767028.588012:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767028.588075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767025/real 1775767028] req@ffffa0e9d0542d80 x1862023861447040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767028.588117:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861447040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767028.588130:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767028.588136:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767028.588141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767028.588157:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767025/real 1775767028] req@ffffa0e9d0540a80 x1862023861447168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767028.588170:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861447168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767028.588200:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767028.588204:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767028.588208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447168:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767028.588217:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767025/real 1775767028] req@ffffa0e9d0541f80 x1862023861446912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767028.588231:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861446912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767028.588238:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767028.588243:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767028.588246:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861446912:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775767029.163612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767029.419660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767030.187644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767030.443560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767030.635647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767030.635674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767030.635678:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767030.635687:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2759 00000100:00080000:1.0:1775767030.635696:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767030.635702:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767030.635746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861447296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767030.635770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767030.635773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767030.635774:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767030.635777:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2759 00000100:00080000:1.0:1775767030.635780:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767030.635782:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767030.635790:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767030.635799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861447424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767030.635819:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767030.635822:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767030.635824:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767030.635827:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2759 00000100:00080000:1.0:1775767030.635829:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767030.635831:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767030.635835:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767030.635844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861447552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767030.635910:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861447296:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767030.636025:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767030.636091:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767030/real 1775767030] req@ffffa0e9c8ded180 x1862023861447296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767030.636114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861447296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767030.636126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767030.636131:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767030.636136:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861447296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767030.636146:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767030.636192:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767030.636219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767030/real 1775767030] req@ffffa0e9c8dec000 x1862023861447424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767030.636235:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861447424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767030.636245:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767030.636248:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767030.636252:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447424:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767030.636260:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767030.636283:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767030.636295:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767030/real 1775767030] req@ffffa0e9c8dee300 x1862023861447552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767030.636308:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861447552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767030.636318:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767030.636321:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767030.636326:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447552:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767031.211665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767031.467618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767032.235751:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767032.491643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767033.259680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767033.515608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767034.283654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767034.539612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767035.307722:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767035.563552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767035.755664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767035.755673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767035.755678:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767035.755688:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2764 00000100:00080000:2.0:1775767035.755700:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767035.755707:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767035.755748:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861447680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767035.755794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767035.755797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767035.755799:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767035.755802:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2764 00000100:00080000:2.0:1775767035.755805:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767035.755809:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767035.755818:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767035.755829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861447808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767035.755837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767035.755840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767035.755842:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767035.755860:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2764 00000100:00100000:0.0:1775767035.755865:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861447680:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767035.755865:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767035.755870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767035.755878:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767035.755887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861447936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767035.755957:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767035.756018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767035/real 1775767035] req@ffffa0e9d0713b80 x1862023861447680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767035.756065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861447680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767035.756078:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767035.756083:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767035.756088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861447680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767035.756098:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767035.756127:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767035.756142:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767035/real 1775767035] req@ffffa0e9d0711c00 x1862023861447808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767035.756157:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861447808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767035.756167:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767035.756171:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767035.756175:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447808:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767035.756208:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767035.756233:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767035.756246:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767035/real 1775767035] req@ffffa0e9d0711f80 x1862023861447936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767035.756260:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861447936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767035.756283:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767035.756287:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767035.756291:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861447936:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767036.331768:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767036.587596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767037.355703:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767037.611591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767038.379639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767038.635576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767039.403657:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767039.659607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767040.427679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767040.683579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767040.875601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767040.875610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767040.875616:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767040.875625:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2769 00000100:00080000:1.0:1775767040.875637:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767040.875643:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767040.875684:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861448064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767040.875726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767040.875731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767040.875733:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767040.875738:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2769 00000100:00080000:1.0:1775767040.875742:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767040.875747:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767040.875760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767040.875775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861448192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767040.875785:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767040.875788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767040.875792:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767040.875794:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861448064:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767040.875796:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2769 00000100:00080000:1.0:1775767040.875801:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767040.875807:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767040.875827:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767040.875837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861448320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767040.875882:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767040.875921:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767040/real 1775767040] req@ffffa0e9c8deed80 x1862023861448064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767040.875937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861448064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767040.875945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767040.875949:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767040.875953:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861448064:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767040.875981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767040.876006:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767040.876019:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767040/real 1775767040] req@ffffa0e9c8ded500 x1862023861448192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767040.876034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861448192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767040.876043:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767040.876054:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767040.876058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767040.876065:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767040.876088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767040.876100:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767040/real 1775767040] req@ffffa0e9c8dec380 x1862023861448320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767040.876114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861448320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767040.876123:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767040.876126:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767040.876130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448320:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767041.451750:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767041.707599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767042.475697:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767042.731574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767043.499641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767043.755599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767044.523725:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767044.779609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767045.547602:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767045.803575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767045.995677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767045.995687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767045.995692:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767045.995702:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2774 00000100:00080000:3.0:1775767045.995714:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767045.995720:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767045.995762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861448448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767045.995807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767045.995815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767045.995818:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767045.995823:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2774 00000100:00080000:3.0:1775767045.995827:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767045.995832:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767045.995844:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767045.995859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861448448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767045.995864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861448576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767045.995875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767045.995878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767045.995880:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767045.995883:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2774 00000100:00080000:3.0:1775767045.995887:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767045.995890:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767045.995894:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767045.995905:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861448704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767045.995935:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767045.995976:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767045/real 1775767045] req@ffffa0e9d0540e00 x1862023861448448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767045.995992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861448448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767045.996001:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767045.996004:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767045.996007:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861448448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767045.996029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767045.996053:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767045.996062:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767045/real 1775767045] req@ffffa0e9d0540700 x1862023861448576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767045.996071:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861448576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767045.996077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767045.996079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767045.996081:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448576:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767045.996085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767045.996095:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767045.996101:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767045/real 1775767045] req@ffffa0e9d0543800 x1862023861448704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767045.996108:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861448704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767045.996113:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767045.996115:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767045.996117:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448704:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767046.571749:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767046.827601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767047.595669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767047.851614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767048.619678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767048.875578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767049.643723:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767049.899565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767050.667664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767050.923561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767051.115613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767051.115622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767051.115628:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767051.115638:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2779 00000100:00080000:3.0:1775767051.115667:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767051.115674:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767051.115711:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861448832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767051.115739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767051.115742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767051.115743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767051.115746:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2779 00000100:00080000:3.0:1775767051.115749:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767051.115751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767051.115760:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767051.115768:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861448960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767051.115776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767051.115780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767051.115781:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767051.115784:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2779 00000100:00080000:3.0:1775767051.115786:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767051.115788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767051.115791:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767051.115800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861449088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767051.115801:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861448832:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767051.115878:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767051.115927:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767051/real 1775767051] req@ffffa0e9d0541c00 x1862023861448832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767051.115951:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861448832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767051.115959:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767051.115963:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767051.115966:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861448832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767051.115972:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767051.115990:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767051.115998:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767051/real 1775767051] req@ffffa0e9d0542300 x1862023861448960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767051.116006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861448960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767051.116012:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767051.116014:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767051.116016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861448960:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767051.116049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767051.116059:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767051.116065:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767051/real 1775767051] req@ffffa0e9d0540380 x1862023861449088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767051.116073:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861449088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767051.116087:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767051.116089:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767051.116092:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449088:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767051.691725:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767051.947560:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767052.715659:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767052.971643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767053.739668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767053.995667:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767054.763663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767055.019622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767055.787654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767056.043593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767056.235597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767056.235609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767056.235614:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767056.235625:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2784 00000100:00080000:2.0:1775767056.235634:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767056.235640:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767056.235673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861449216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767056.235705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767056.235708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767056.235709:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767056.235712:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2784 00000100:00080000:2.0:1775767056.235714:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767056.235717:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767056.235724:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767056.235732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861449344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767056.235739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767056.235742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767056.235759:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767056.235762:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2784 00000100:00080000:2.0:1775767056.235766:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767056.235771:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861449216:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767056.235772:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767056.235779:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767056.235789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861449472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767056.235839:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767056.235899:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767056/real 1775767056] req@ffffa0e9d0710700 x1862023861449216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767056.235920:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861449216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767056.235932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767056.235937:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767056.235942:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861449216:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767056.235950:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767056.235974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767056.235986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767056/real 1775767056] req@ffffa0e9d0712d80 x1862023861449344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767056.236000:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861449344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767056.236010:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767056.236020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767056.236025:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449344:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767056.236049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767056.236071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767056.236083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767056/real 1775767056] req@ffffa0e9d0712680 x1862023861449472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767056.236096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861449472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767056.236104:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767056.236108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767056.236112:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449472:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767056.811687:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767057.067591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767057.835649:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767058.091647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767058.859699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767059.115625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767059.883734:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767060.139650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767060.907612:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767061.163622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767061.355661:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767061.355674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767061.355679:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767061.355690:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2789 00000100:00080000:1.0:1775767061.355700:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767061.355707:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767061.355748:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861449600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767061.355786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767061.355790:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767061.355792:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767061.355797:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2789 00000100:00080000:1.0:1775767061.355801:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767061.355806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767061.355816:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767061.355832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861449728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767061.355834:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861449600:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767061.355867:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767061.355871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767061.355873:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767061.355878:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2789 00000100:00080000:1.0:1775767061.355884:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767061.355889:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775767061.355893:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775767061.355899:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767061.355914:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861449856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767061.355936:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767061/real 1775767061] req@ffffa0e9c8def800 x1862023861449600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767061.355952:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861449600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767061.355960:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767061.355964:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767061.355966:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861449600:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767061.355986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767061.356009:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767061.356018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767061/real 1775767061] req@ffffa0e9c8def480 x1862023861449728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767061.356027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861449728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767061.356032:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767061.356035:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767061.356037:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449728:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767061.356040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767061.356052:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767061.356062:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767061/real 1775767061] req@ffffa0e9c8dedc00 x1862023861449856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767061.356075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861449856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767061.356083:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767061.356086:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767061.356090:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861449856:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767061.931688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767062.187611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767062.955654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767063.211612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767063.979641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767064.235557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767065.003690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767065.259616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767066.027641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767066.283583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767066.475555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767066.475564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767066.475569:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767066.475578:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2794 00000100:00080000:1.0:1775767066.475588:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767066.475594:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767066.475631:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861449984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767066.475656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767066.475659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767066.475660:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767066.475663:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2794 00000100:00080000:1.0:1775767066.475665:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767066.475688:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767066.475697:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767066.475702:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861449984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767066.475710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861450112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767066.475719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767066.475722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767066.475724:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767066.475727:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2794 00000100:00080000:1.0:1775767066.475730:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767066.475732:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767066.475736:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767066.475743:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861450240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767066.475769:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767066.475821:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767066/real 1775767066] req@ffffa0e9c8deea00 x1862023861449984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767066.475844:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861449984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767066.475852:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767066.475855:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767066.475858:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861449984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767066.475877:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767066.475895:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767066.475904:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767066/real 1775767066] req@ffffa0e9c8dee680 x1862023861450112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767066.475912:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861450112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767066.475917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767066.475919:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767066.475922:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450112:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767066.475925:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767066.475935:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767066.475940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767066/real 1775767066] req@ffffa0e9c8ded880 x1862023861450240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767066.475948:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861450240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767066.475962:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767066.475964:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767066.475966:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450240:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767067.051658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767067.307595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767068.075629:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767068.331590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767069.099660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767069.355623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767070.123629:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767070.379577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767071.147690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767071.403584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767071.595568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767071.595577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767071.595583:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767071.595593:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2800 00000100:00080000:1.0:1775767071.595606:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767071.595612:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767071.595655:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861450368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767071.595681:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767071.595698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767071.595700:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767071.595704:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2800 00000100:00080000:1.0:1775767071.595708:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767071.595711:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767071.595721:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767071.595731:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861450368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767071.595734:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861450496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767071.595745:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767071.595747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767071.595749:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767071.595752:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2800 00000100:00080000:1.0:1775767071.595755:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767071.595757:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767071.595760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767071.595767:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861450624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767071.595806:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767071.595861:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767071/real 1775767071] req@ffffa0e9c8deca80 x1862023861450368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767071.595876:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861450368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767071.595886:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767071.595890:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767071.595894:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861450368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767071.595915:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767071.595931:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767071.595940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767071/real 1775767071] req@ffffa0e9c8dedf80 x1862023861450496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767071.595948:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861450496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767071.595954:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767071.595962:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767071.595965:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450496:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767071.595969:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767071.595983:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767071.595997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767071/real 1775767071] req@ffffa0e9c8dece00 x1862023861450624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767071.596006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861450624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767071.596011:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767071.596014:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767071.596016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450624:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767072.171686:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767072.427585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767073.195659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767073.451619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767074.219657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767074.475580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767075.243698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767075.499607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767076.267617:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767076.523588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767076.715611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767076.715623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767076.715629:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767076.715641:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2805 00000100:00080000:0.0:1775767076.715655:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767076.715664:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767076.715715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861450752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767076.715752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767076.715756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767076.715759:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767076.715784:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2805 00000100:00100000:1.0:1775767076.715787:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861450752:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767076.715790:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767076.715799:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767076.715812:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767076.715830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861450880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767076.715845:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767076.715849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767076.715851:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767076.715856:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2805 00000100:00080000:0.0:1775767076.715869:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767076.715874:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:1.0:1775767076.715880:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:0.0:1775767076.715884:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767076.715902:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861451008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767076.715928:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767076/real 1775767076] req@ffffa0e9d09df100 x1862023861450752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767076.715958:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861450752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767076.715967:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767076.715970:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767076.715973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861450752:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767076.715996:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767076.716014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767076.716023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767076/real 1775767076] req@ffffa0e9d09dfb80 x1862023861450880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767076.716031:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861450880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767076.716037:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767076.716040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767076.716042:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861450880:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767076.716045:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767076.716055:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767076.716061:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767076/real 1775767076] req@ffffa0e9d09ded80 x1862023861451008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767076.716068:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861451008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767076.716082:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767076.716085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767076.716087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451008:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767077.291688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767077.547572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767078.315728:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767078.571622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767079.339646:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767079.595575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767080.363815:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767080.619614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767081.387737:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767081.643602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767081.835639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767081.835663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767081.835667:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767081.835695:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2810 00000100:00080000:3.0:1775767081.835708:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767081.835715:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767081.835755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861451136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767081.835785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767081.835788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767081.835790:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767081.835793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2810 00000100:00080000:3.0:1775767081.835796:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767081.835798:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767081.835808:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767081.835819:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861451264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767081.835827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767081.835831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767081.835833:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767081.835836:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2810 00000100:00080000:3.0:1775767081.835839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767081.835841:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767081.835844:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767081.835867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861451392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767081.835884:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861451136:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767081.835964:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767081.836021:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767081/real 1775767081] req@ffffa0e9d0540000 x1862023861451136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767081.836045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861451136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767081.836058:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767081.836063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767081.836067:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861451136:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767081.836076:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767081.836101:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767081.836129:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767081/real 1775767081] req@ffffa0e9d0542a00 x1862023861451264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767081.836143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861451264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767081.836152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767081.836156:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767081.836169:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767081.836197:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767081.836222:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767081.836234:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767081/real 1775767081] req@ffffa0e9d0543b80 x1862023861451392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767081.836247:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861451392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767081.836256:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767081.836260:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767081.836264:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451392:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767082.411641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767082.667595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767083.435652:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767083.691628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767084.459684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767084.715617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767085.483641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767085.739607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767086.507681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767086.763550:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767086.955599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767086.955609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767086.955613:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767086.955622:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2815 00000100:00080000:3.0:1775767086.955631:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767086.955637:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767086.955675:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861451520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767086.955702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767086.955705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767086.955706:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767086.955709:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2815 00000100:00080000:3.0:1775767086.955712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767086.955714:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767086.955722:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767086.955731:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861451648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767086.955739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767086.955741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767086.955743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767086.955746:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2815 00000100:00080000:3.0:1775767086.955749:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767086.955751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767086.955775:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767086.955785:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861451520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767086.955786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861451776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767086.955902:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767086.955958:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767086/real 1775767086] req@ffffa0e9d0541500 x1862023861451520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767086.955977:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861451520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767086.955986:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767086.955989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767086.955993:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861451520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767086.956018:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767086.956046:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767086.956060:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767086/real 1775767086] req@ffffa0e9d0541880 x1862023861451648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767086.956071:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861451648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767086.956077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767086.956079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767086.956082:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451648:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767086.956129:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767086.956146:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767086.956165:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767086/real 1775767086] req@ffffa0e9d0542680 x1862023861451776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767086.956174:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861451776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767086.956179:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767086.956182:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767086.956184:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861451776:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767087.531709:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767087.787616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767088.555679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767088.811600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767089.579636:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767089.835562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767090.603681:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767090.859606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767091.627618:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767091.883567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767092.075616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767092.075627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767092.075633:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767092.075645:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2820 00000100:00080000:3.0:1775767092.075660:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767092.075669:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767092.075746:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861451904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767092.075792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767092.075800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767092.075804:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767092.075812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2820 00000100:00080000:3.0:1775767092.075820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767092.075829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767092.075846:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767092.075881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861452032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767092.075890:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861451904:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767092.075892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767092.075895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767092.075897:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767092.075900:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2820 00000100:00080000:3.0:1775767092.075920:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767092.075924:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767092.075929:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767092.075939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861452160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767092.076012:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767092.076040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452160:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767092.651651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767092.907585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767093.675744:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767093.931574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767094.699624:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767094.955602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775767095.147770:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775767095.147816:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767095.147835:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767095.147879:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767095.147890:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775767095.147954:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767092/real 1775767095] req@ffffa0e9d0541880 x1862023861452032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767095.147992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861452032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767095.148006:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767095.148013:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767095.148018:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767095.148027:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767092/real 1775767095] req@ffffa0e9d0541500 x1862023861452160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767095.148043:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861452160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767095.148052:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767095.148057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767095.148061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452160:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767095.148083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767092/real 1775767095] req@ffffa0e9d0542680 x1862023861451904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767095.148121:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861451904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767095.148152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767095.148157:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767095.148161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861451904:192.168.202.104@tcp:250:kworker.0 00010000:00010000:0.0:1775767095.723730:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767095.979601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767096.747671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767097.003597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767097.195648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767097.195659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767097.195664:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767097.195675:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2825 00000100:00080000:2.0:1775767097.195684:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767097.195691:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767097.195726:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861452288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767097.195770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767097.195772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767097.195788:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767097.195791:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2825 00000100:00080000:2.0:1775767097.195794:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767097.195797:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767097.195806:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767097.195815:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861452416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767097.195823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767097.195826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767097.195828:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767097.195830:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861452288:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767097.195832:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2825 00000100:00080000:2.0:1775767097.195836:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767097.195841:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767097.195844:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767097.195851:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861452544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767097.195899:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767097.195944:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767097/real 1775767097] req@ffffa0e9d0710a80 x1862023861452288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767097.195960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861452288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767097.195968:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767097.195972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767097.195975:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861452288:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767097.196001:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767097.196024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767097.196037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767097/real 1775767097] req@ffffa0e9d0710e00 x1862023861452416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767097.196049:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861452416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767097.196069:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767097.196073:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767097.196076:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452416:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767097.196082:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767097.196107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767097.196134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767097/real 1775767097] req@ffffa0e9d0712a00 x1862023861452544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767097.196145:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861452544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767097.196152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767097.196155:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767097.196158:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452544:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767097.771693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767098.027594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767098.795623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767099.051577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767099.819703:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767100.075602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767100.843710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767101.099672:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767101.867684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767102.123651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767102.315606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767102.315615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767102.315620:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767102.315629:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2830 00000100:00080000:2.0:1775767102.315639:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767102.315644:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767102.315679:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861452672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767102.315725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767102.315728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767102.315729:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767102.315732:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2830 00000100:00080000:2.0:1775767102.315735:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767102.315738:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767102.315746:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767102.315755:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861452800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767102.315762:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767102.315764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767102.315766:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767102.315768:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2830 00000100:00080000:2.0:1775767102.315770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767102.315772:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767102.315775:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767102.315783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861452928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767102.315784:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861452672:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767102.315847:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767102.315892:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767102/real 1775767102] req@ffffa0e9d0713480 x1862023861452672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767102.315907:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861452672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767102.315926:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767102.315929:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767102.315932:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861452672:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767102.315937:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767102.315956:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767102.315965:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767102/real 1775767102] req@ffffa0e9d0711500 x1862023861452800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767102.315974:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861452800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767102.315979:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767102.315981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767102.315983:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452800:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767102.316002:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767102.316013:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767102.316023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767102/real 1775767102] req@ffffa0e9d0711880 x1862023861452928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767102.316036:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861452928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767102.316044:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767102.316047:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767102.316051:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861452928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767102.891657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767103.147652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767103.915654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767104.171623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767104.939657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767105.195625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767105.963684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767106.219622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767106.987645:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767107.243574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767107.435591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767107.435601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767107.435605:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767107.435614:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2835 00000100:00080000:0.0:1775767107.435623:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767107.435628:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767107.435669:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861453056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767107.435735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767107.435738:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767107.435740:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767107.435743:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2835 00000100:00080000:0.0:1775767107.435746:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767107.435749:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767107.435758:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767107.435770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861453184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767107.435779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861453056:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767107.435780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767107.435788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767107.435790:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767107.435799:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2835 00000100:00080000:0.0:1775767107.435801:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767107.435804:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767107.435808:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767107.435818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861453312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767107.435853:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767107.435900:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767107/real 1775767107] req@ffffa0e9d09dc700 x1862023861453056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767107.435931:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861453056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767107.435940:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767107.435944:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767107.435947:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861453056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767107.435966:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767107.435985:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767107.435994:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767107/real 1775767107] req@ffffa0e9d09df800 x1862023861453184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767107.436018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861453184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767107.436029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767107.436032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767107.436036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453184:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767107.436054:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767107.436077:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767107.436088:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767107/real 1775767107] req@ffffa0e9d09dd880 x1862023861453312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767107.436096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861453312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767107.436101:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767107.436104:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767107.436106:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453312:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767108.011692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767108.267580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767109.035648:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767109.291638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767110.059664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767110.315649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767111.083840:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767111.339555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767112.107684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767112.363619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767112.555566:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767112.555577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767112.555581:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767112.555590:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2841 00000100:00080000:0.0:1775767112.555599:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767112.555605:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767112.555646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861453440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767112.555677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767112.555680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767112.555681:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767112.555684:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2841 00000100:00080000:0.0:1775767112.555687:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767112.555690:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767112.555699:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767112.555709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861453568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767112.555716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767112.555719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767112.555720:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767112.555724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2841 00000100:00080000:0.0:1775767112.555727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767112.555729:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767112.555745:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861453440:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767112.555755:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767112.555765:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861453696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767112.555806:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767112.555853:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767112/real 1775767112] req@ffffa0e9d09dd500 x1862023861453440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767112.555868:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861453440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767112.555875:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767112.555878:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767112.555882:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861453440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767112.555886:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767112.555917:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767112.555926:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767112/real 1775767112] req@ffffa0e9d09dd180 x1862023861453568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767112.555934:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861453568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767112.555939:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767112.555942:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767112.555944:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453568:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767112.555994:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767112.556010:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767112.556018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767112/real 1775767112] req@ffffa0e9d09ddc00 x1862023861453696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767112.556026:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861453696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767112.556031:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767112.556033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767112.556035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453696:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767113.131689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767113.387555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767114.155681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767114.411600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767115.179701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767115.435647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767116.203660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767116.459600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767117.227755:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767117.483607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767117.675632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767117.675646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767117.675651:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767117.675662:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2846 00000100:00080000:2.0:1775767117.675698:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767117.675745:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767117.675783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861453824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767117.675816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767117.675820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767117.675822:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767117.675827:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2846 00000100:00080000:2.0:1775767117.675831:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767117.675835:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767117.675846:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767117.675861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861453952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767117.675867:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861453824:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767117.675876:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767117.675879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767117.675880:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767117.675884:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2846 00000100:00080000:2.0:1775767117.675887:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767117.675889:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767117.675894:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767117.675922:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861454080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767117.675939:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767117.675983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767117/real 1775767117] req@ffffa0e9d0713100 x1862023861453824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767117.676003:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861453824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767117.676014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767117.676019:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767117.676023:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861453824:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767117.676048:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767117.676071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767117.676080:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767117/real 1775767117] req@ffffa0e9d0712300 x1862023861453952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767117.676089:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861453952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767117.676095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767117.676097:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767117.676099:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861453952:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767117.676103:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767117.676113:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767117.676119:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767117/real 1775767117] req@ffffa0e9d0711880 x1862023861454080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767117.676127:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861454080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767117.676141:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767117.676143:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767117.676145:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767118.251626:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767118.507597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767119.275679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767119.531619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767120.299703:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767120.555583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767121.323695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767121.579624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767122.347717:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767122.603630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767122.795598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767122.795609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767122.795615:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767122.795626:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2851 00000100:00080000:0.0:1775767122.795641:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767122.795650:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767122.795694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861454208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767122.795727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767122.795731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767122.795733:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767122.795752:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2851 00000100:00080000:0.0:1775767122.795757:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767122.795761:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767122.795773:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767122.795784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861454336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767122.795795:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767122.795799:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767122.795801:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767122.795805:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2851 00000100:00080000:0.0:1775767122.795809:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767122.795814:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861454208:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767122.795817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767122.795826:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767122.795837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861454464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767122.795905:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767122.795946:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767122/real 1775767122] req@ffffa0e9d09ddf80 x1862023861454208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767122.795962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861454208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767122.795970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767122.795974:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767122.795977:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861454208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767122.795982:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767122.795997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767122.796005:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767122/real 1775767122] req@ffffa0e9d09dca80 x1862023861454336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767122.796013:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861454336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767122.796030:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767122.796032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767122.796035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767122.796056:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767122.796070:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767122.796076:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767122/real 1775767122] req@ffffa0e9d09de680 x1862023861454464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767122.796084:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861454464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767122.796089:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767122.796091:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767122.796092:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454464:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767123.371672:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767123.627577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767124.395654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767124.651598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767125.419907:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767125.675682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767126.443622:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767126.699604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767127.467652:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767127.723585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767127.915602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767127.915611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767127.915615:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767127.915625:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2856 00000100:00080000:3.0:1775767127.915635:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767127.915641:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767127.915679:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861454592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767127.915721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767127.915725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767127.915726:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767127.915730:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2856 00000100:00080000:3.0:1775767127.915732:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767127.915735:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767127.915745:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767127.915755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861454720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767127.915762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767127.915829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767127.915831:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767127.915836:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2856 00000100:00100000:1.0:1775767127.915842:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861454592:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767127.915842:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767127.915851:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767127.915856:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767127.915869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861454848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767127.915934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767127.915976:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767127/real 1775767127] req@ffffa0e9d0543b80 x1862023861454592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767127.915993:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861454592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767127.916022:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767127.916038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767127.916041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861454592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767127.916047:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767127.916067:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767127.916077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767127/real 1775767127] req@ffffa0e9d0542a00 x1862023861454720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767127.916087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861454720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767127.916093:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767127.916096:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767127.916098:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454720:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767127.916120:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767127.916132:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767127.916139:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767127/real 1775767127] req@ffffa0e9d0540000 x1862023861454848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767127.916147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861454848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767127.916153:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767127.916155:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767127.916157:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861454848:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767128.491634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767128.747629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767129.515788:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767129.771627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767130.539642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767130.795619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767131.563672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767131.819670:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767132.587685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767132.843670:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767133.035627:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767133.035652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767133.035657:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767133.035667:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2861 00000100:00080000:2.0:1775767133.035679:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767133.035685:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767133.035723:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861454976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767133.035772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767133.035776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767133.035778:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767133.035783:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2861 00000100:00080000:2.0:1775767133.035787:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767133.035792:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767133.035804:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767133.035821:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861455104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767133.035822:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861454976:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767133.035838:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767133.035841:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767133.035843:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767133.035845:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2861 00000100:00080000:2.0:1775767133.035849:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767133.035852:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767133.035856:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767133.035864:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861455232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767133.035902:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767133.035943:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767133/real 1775767133] req@ffffa0e9d0711500 x1862023861454976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767133.035959:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861454976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767133.035967:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767133.035970:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767133.035973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861454976:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767133.035997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767133.036011:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767133.036019:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767133/real 1775767133] req@ffffa0e9d0713480 x1862023861455104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767133.036027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861455104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767133.036032:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767133.036034:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767133.036036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455104:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767133.036040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767133.036061:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767133.036067:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767133/real 1775767133] req@ffffa0e9d0712a00 x1862023861455232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767133.036075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861455232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767133.036080:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767133.036082:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767133.036084:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455232:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767133.611665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767133.867610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767134.635675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767134.891602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767135.659733:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767135.915618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767136.683683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767136.939612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767137.707733:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767137.963593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767138.155620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767138.155632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767138.155638:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767138.155651:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2866 00000100:00080000:2.0:1775767138.155712:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767138.155721:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767138.155770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861455360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767138.155805:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767138.155810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767138.155812:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767138.155816:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2866 00000100:00080000:2.0:1775767138.155821:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767138.155826:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767138.155838:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767138.155858:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861455488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767138.155871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767138.155875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767138.155877:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767138.155884:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2866 00000100:00100000:1.0:1775767138.155895:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861455360:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767138.155895:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767138.155909:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767138.155944:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767138.155964:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861455616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767138.155999:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767138.156047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767138/real 1775767138] req@ffffa0e9d0710e00 x1862023861455360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767138.156064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861455360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767138.156075:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767138.156079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767138.156082:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861455360:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767138.156089:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767138.156124:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767138.156134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767138/real 1775767138] req@ffffa0e9d0710a80 x1862023861455488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767138.156144:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861455488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767138.156150:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767138.156153:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767138.156156:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455488:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767138.156179:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767138.156207:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767138.156214:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767138/real 1775767138] req@ffffa0e9d0712680 x1862023861455616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767138.156221:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861455616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767138.156226:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767138.156228:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767138.156230:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455616:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767138.731694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767138.987603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767139.755754:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767140.011741:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767140.779694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767141.035609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767141.803768:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767142.059668:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767142.827683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767143.083616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767143.275598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767143.275608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767143.275613:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767143.275623:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2871 00000100:00080000:2.0:1775767143.275653:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767143.275677:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767143.275747:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861455744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767143.275775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767143.275778:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767143.275780:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767143.275783:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2871 00000100:00080000:2.0:1775767143.275786:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767143.275788:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767143.275798:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767143.275807:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861455872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767143.275820:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767143.275824:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767143.275838:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767143.275844:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861455744:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767143.275847:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2871 00000100:00080000:2.0:1775767143.275871:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767143.275877:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767143.275885:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767143.275907:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861456000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767143.275927:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767143.275986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767143/real 1775767143] req@ffffa0e9d0712d80 x1862023861455744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767143.276004:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861455744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767143.276013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767143.276016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767143.276020:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861455744:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767143.276026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767143.276044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767143.276053:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767143/real 1775767143] req@ffffa0e9d0710700 x1862023861455872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767143.276078:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861455872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767143.276084:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767143.276086:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767143.276088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861455872:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767143.276114:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767143.276128:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767143.276135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767143/real 1775767143] req@ffffa0e9d0711f80 x1862023861456000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767143.276144:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861456000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767143.276163:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767143.276165:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767143.276168:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456000:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767143.851659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767144.107604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767144.875719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767145.131725:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767145.899736:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767146.155582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767146.923717:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767147.179643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767147.947795:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767148.203741:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767148.395697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767148.395711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767148.395719:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767148.395747:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2876 00000100:00080000:3.0:1775767148.395764:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767148.395774:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767148.395834:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861456128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767148.395875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767148.395880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767148.395883:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767148.395888:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2876 00000100:00080000:3.0:1775767148.395892:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767148.395897:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767148.395922:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767148.395939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861456256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767148.395976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767148.395980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767148.395983:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767148.395988:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2876 00000100:00080000:3.0:1775767148.395993:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767148.395998:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767148.396004:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767148.396016:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861456384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767148.396171:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861456128:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767148.396269:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767148.396329:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767148/real 1775767148] req@ffffa0e9d0540380 x1862023861456128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767148.396356:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861456128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767148.396381:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767148.396387:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767148.396392:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861456128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767148.396402:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767148.396428:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767148.396442:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767148/real 1775767148] req@ffffa0e9d0542300 x1862023861456256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767148.396463:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861456256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767148.396574:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767148.396613:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767148.396618:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767148.396628:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767148.396661:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767148.396685:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767148/real 1775767148] req@ffffa0e9d0541c00 x1862023861456384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767148.396700:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861456384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767148.396711:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767148.396715:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767148.396718:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456384:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767148.971741:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767149.227626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767149.995743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767150.251610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767151.019712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767151.275616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767152.043810:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767152.299585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767153.067722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767153.323648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767153.515642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767153.515652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767153.515657:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767153.515666:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2882 00000100:00080000:3.0:1775767153.515676:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767153.515682:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767153.515733:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861456512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767153.515764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767153.515768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767153.515770:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767153.515774:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2882 00000100:00080000:3.0:1775767153.515778:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767153.515782:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767153.515829:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767153.515847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861456640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767153.515856:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861456512:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767153.515876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767153.515895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767153.515897:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767153.515901:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2882 00000100:00080000:3.0:1775767153.515905:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767153.515908:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767153.515914:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767153.515928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861456768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767153.515954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767153.515999:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767153/real 1775767153] req@ffffa0e9d0543800 x1862023861456512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767153.516017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861456512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767153.516053:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767153.516057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767153.516061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861456512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767153.516088:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767153.516107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767153.516133:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767153/real 1775767153] req@ffffa0e9d0540700 x1862023861456640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767153.516141:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861456640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767153.516147:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767153.516150:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767153.516152:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456640:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767153.516156:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767153.516168:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767153.516174:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767153/real 1775767153] req@ffffa0e9d0540e00 x1862023861456768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767153.516181:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861456768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767153.516198:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767153.516200:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767153.516202:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861456768:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767154.091725:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767154.347616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767155.115747:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767155.371628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767156.139699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767156.395630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767157.163717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767157.419615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767158.187693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767158.443584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767158.635616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767158.635625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767158.635631:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767158.635642:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2887 00000100:00080000:3.0:1775767158.635656:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767158.635663:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767158.635707:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861456896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767158.635737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767158.635740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767158.635742:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767158.635746:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2887 00000100:00080000:3.0:1775767158.635769:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767158.635773:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767158.635784:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767158.635797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861457024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767158.635807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767158.635811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767158.635813:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767158.635817:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2887 00000100:00080000:3.0:1775767158.635819:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767158.635822:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767158.635825:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767158.635836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861457152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767158.635837:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861456896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767158.635953:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767158.635983:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457152:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767159.211654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767159.467634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767160.235747:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767160.491634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767161.259708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767161.515594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775767161.707894:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775767161.707963:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767161.707988:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767161.708090:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767161.708101:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775767161.708202:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767158/real 1775767161] req@ffffa0e9d0542d80 x1862023861457152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767161.708238:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861457152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767161.708251:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767161.708256:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767161.708262:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457152:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767161.708272:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767158/real 1775767161] req@ffffa0e9d0541f80 x1862023861456896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767161.708285:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861456896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767161.708292:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767161.708297:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767161.708300:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861456896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767161.708328:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767158/real 1775767161] req@ffffa0e9d0540a80 x1862023861457024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767161.708344:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861457024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767161.708352:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767161.708356:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767161.708360:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457024:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767162.283708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767162.539606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767163.307651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767163.563679:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767163.755617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767163.755627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767163.755632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767163.755643:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2892 00000100:00080000:1.0:1775767163.755655:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767163.755661:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767163.755701:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861457280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767163.755737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767163.755741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767163.755743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767163.755747:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2892 00000100:00080000:1.0:1775767163.755751:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767163.755756:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767163.755770:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767163.755806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861457408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767163.755818:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861457280:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767163.755830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767163.755835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767163.755837:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767163.755842:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2892 00000100:00080000:1.0:1775767163.755850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767163.755855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767163.755863:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767163.755880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861457536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767163.755905:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767163.755983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767163/real 1775767163] req@ffffa0e9c8dec700 x1862023861457280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767163.756006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861457280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767163.756020:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767163.756027:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767163.756031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861457280:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767163.756058:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767163.756085:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767163.756099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767163/real 1775767163] req@ffffa0e9c8defb80 x1862023861457408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767163.756111:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861457408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767163.756141:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767163.756146:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767163.756149:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457408:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767163.756155:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767163.756178:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767163.756191:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767163/real 1775767163] req@ffffa0e9c8def100 x1862023861457536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767163.756203:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861457536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767163.756210:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767163.756214:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767163.756216:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457536:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767164.331640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767164.587654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767165.355710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767165.611627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767166.379666:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767166.635627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767167.403667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767167.659624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767168.427737:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767168.683687:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767168.875630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767168.875643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767168.875650:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767168.875663:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2897 00000100:00080000:1.0:1775767168.875679:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767168.875689:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767168.875747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861457664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767168.875789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767168.875824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767168.875828:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767168.875838:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2897 00000100:00080000:1.0:1775767168.875847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767168.875856:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767168.875874:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767168.875891:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861457664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767168.875906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861457792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767168.875932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767168.875937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767168.875940:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767168.875946:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2897 00000100:00080000:1.0:1775767168.875953:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767168.875958:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767168.875966:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767168.875983:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861457920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767168.876001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767168.876061:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767168/real 1775767168] req@ffffa0e9c8def100 x1862023861457664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767168.876135:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861457664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767168.876150:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767168.876156:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767168.876162:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861457664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767168.876195:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767168.876228:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767168.876245:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767168/real 1775767168] req@ffffa0e9c8defb80 x1862023861457792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767168.876260:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861457792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767168.876295:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767168.876299:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767168.876303:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457792:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767168.876309:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767168.876330:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767168.876404:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767168/real 1775767168] req@ffffa0e9c8dec700 x1862023861457920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767168.876422:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861457920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767168.876448:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767168.876453:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767168.876457:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861457920:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767169.451669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767169.707617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767170.475686:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767170.731624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767171.499708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767171.755666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767172.523673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767172.779656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767173.547658:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767173.803633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767173.995661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767173.995670:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767173.995675:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767173.995686:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2902 00000100:00080000:0.0:1775767173.995698:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767173.995704:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767173.995748:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861458048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767173.995776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767173.995779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767173.995781:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767173.995784:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2902 00000100:00080000:0.0:1775767173.995786:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767173.995789:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767173.995814:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767173.995819:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861458048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767173.995827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861458176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767173.995837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767173.995839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767173.995840:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767173.995844:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2902 00000100:00080000:0.0:1775767173.995847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767173.995849:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767173.995853:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767173.995863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861458304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767173.995897:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767173.995937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767173/real 1775767173] req@ffffa0e9d09de680 x1862023861458048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767173.995964:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861458048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767173.995973:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767173.995977:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767173.995980:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861458048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767173.996011:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767173.996029:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767173.996038:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767173/real 1775767173] req@ffffa0e9d09dca80 x1862023861458176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767173.996047:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861458176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767173.996053:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767173.996055:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767173.996057:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767173.996061:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767173.996071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767173.996077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767173/real 1775767173] req@ffffa0e9d09ddf80 x1862023861458304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767173.996085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861458304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767173.996099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767173.996102:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767173.996104:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458304:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767174.571683:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767174.827569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767175.595710:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767175.851612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767176.619668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767176.875636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767177.643704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767177.899596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767178.667716:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767178.923620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767179.115624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767179.115636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767179.115642:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767179.115655:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2907 00000100:00080000:0.0:1775767179.115670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767179.115679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767179.115743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861458432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767179.115780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767179.115789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767179.115792:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767179.115796:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2907 00000100:00080000:0.0:1775767179.115801:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767179.115806:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767179.115818:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767179.115836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861458560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767179.115836:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861458432:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767179.115855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767179.115859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767179.115861:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767179.115866:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2907 00000100:00080000:0.0:1775767179.115870:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767179.115875:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767179.115881:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767179.115893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861458688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767179.115919:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767179.115961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767179/real 1775767179] req@ffffa0e9d09ddc00 x1862023861458432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767179.115978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861458432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767179.115987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767179.115990:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767179.115993:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861458432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767179.116014:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767179.116031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767179.116039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767179/real 1775767179] req@ffffa0e9d09dd180 x1862023861458560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767179.116047:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861458560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767179.116064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767179.116066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767179.116069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458560:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767179.116073:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767179.116087:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767179.116094:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767179/real 1775767179] req@ffffa0e9d09dd500 x1862023861458688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767179.116102:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861458688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767179.116107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767179.116109:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767179.116110:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458688:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767179.691724:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767179.947651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767180.715655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767180.971608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767181.739665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767181.995593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767182.763667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767183.019614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767183.787678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767184.043612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767184.235610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767184.235619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767184.235624:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767184.235636:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2912 00000100:00080000:0.0:1775767184.235658:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767184.235666:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767184.235715:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861458816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767184.235745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767184.235748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767184.235749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767184.235752:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2912 00000100:00080000:0.0:1775767184.235755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767184.235758:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767184.235767:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767184.235776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861458944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767184.235785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767184.235788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767184.235789:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767184.235794:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861458816:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767184.235795:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2912 00000100:00080000:0.0:1775767184.235800:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767184.235804:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767184.235824:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767184.235835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861459072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767184.235870:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767184.235910:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767184/real 1775767184] req@ffffa0e9d09dd880 x1862023861458816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767184.235929:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861458816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767184.235949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767184.235952:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767184.235955:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861458816:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767184.235979:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767184.235996:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767184.236006:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767184/real 1775767184] req@ffffa0e9d09df800 x1862023861458944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767184.236014:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861458944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767184.236020:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767184.236022:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767184.236025:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861458944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767184.236028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767184.236039:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767184.236044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767184/real 1775767184] req@ffffa0e9d09dc700 x1862023861459072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767184.236052:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861459072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767184.236057:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767184.236059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767184.236061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459072:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767184.811649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767185.067607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767185.835647:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767186.091646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767186.859678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767187.115645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767187.883706:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767188.139650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767188.907680:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767189.163669:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767189.355657:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767189.355666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767189.355670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767189.355680:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2917 00000100:00080000:0.0:1775767189.355692:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767189.355698:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767189.355741:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861459200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767189.355770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767189.355773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767189.355774:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767189.355778:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2917 00000100:00080000:0.0:1775767189.355780:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767189.355783:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767189.355793:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767189.355801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861459328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767189.355829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767189.355833:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767189.355834:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767189.355834:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861459200:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767189.355840:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2917 00000100:00080000:0.0:1775767189.355846:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767189.355849:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767189.355853:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767189.355863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861459456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767189.355914:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767189.355957:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767189/real 1775767189] req@ffffa0e9d09ded80 x1862023861459200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767189.355976:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861459200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767189.355984:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767189.355988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767189.355993:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861459200:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767189.356020:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767189.356037:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767189.356045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767189/real 1775767189] req@ffffa0e9d09dfb80 x1862023861459328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767189.356053:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861459328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767189.356059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767189.356061:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767189.356063:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459328:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767189.356067:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767189.356088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767189.356095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767189/real 1775767189] req@ffffa0e9d09df100 x1862023861459456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767189.356102:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861459456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767189.356107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767189.356109:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767189.356111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459456:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767189.931666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767190.187697:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767190.955662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767191.211708:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767191.979649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767192.235667:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767193.003644:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767193.259619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767194.027685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767194.283648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767194.475611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767194.475619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767194.475624:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767194.475635:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2922 00000100:00080000:0.0:1775767194.475646:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767194.475652:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767194.475694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861459584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767194.475724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767194.475727:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767194.475728:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767194.475731:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2922 00000100:00080000:0.0:1775767194.475751:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767194.475756:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767194.475769:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767194.475773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861459584:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767194.475788:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861459712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767194.475802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767194.475806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767194.475808:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767194.475812:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2922 00000100:00080000:0.0:1775767194.475815:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767194.475819:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767194.475824:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767194.475833:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861459840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767194.475856:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767194.475898:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767194/real 1775767194] req@ffffa0e9d09dce00 x1862023861459584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767194.475914:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861459584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767194.475922:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767194.475925:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767194.475928:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861459584:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767194.475949:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767194.475965:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767194.475981:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767194/real 1775767194] req@ffffa0e9d09dc380 x1862023861459712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767194.475990:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861459712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767194.475996:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767194.475998:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767194.476001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459712:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767194.476005:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767194.476017:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767194.476023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767194/real 1775767194] req@ffffa0e9d09dc000 x1862023861459840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767194.476031:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861459840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767194.476036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767194.476037:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767194.476039:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861459840:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767195.051663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767195.307605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767196.075659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767196.331576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767197.099747:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767197.355583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767198.123668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767198.379570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767199.147689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767199.403583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767199.595614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767199.595625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767199.595629:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767199.595649:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2928 00000100:00080000:0.0:1775767199.595658:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767199.595664:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767199.595701:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861459968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767199.595726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767199.595745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767199.595747:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767199.595751:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2928 00000100:00080000:0.0:1775767199.595754:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767199.595757:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767199.595767:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767199.595774:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861459968:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767199.595780:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861460096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767199.595789:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767199.595791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767199.595793:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767199.595796:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2928 00000100:00080000:0.0:1775767199.595798:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767199.595800:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767199.595803:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767199.595811:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861460224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767199.595833:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767199.595879:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767199/real 1775767199] req@ffffa0e9d09de300 x1862023861459968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767199.595896:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861459968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767199.595906:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767199.595911:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767199.595915:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861459968:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767199.595935:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767199.595951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767199.595960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767199/real 1775767199] req@ffffa0e9d09dea00 x1862023861460096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767199.595968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861460096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767199.595973:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767199.595976:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767199.595978:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460096:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767199.595981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460224:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767199.596001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767199.596008:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767199/real 1775767199] req@ffffa0e9d09df480 x1862023861460224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767199.596016:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861460224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767199.596032:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767199.596034:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767199.596036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460224:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767200.171714:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767200.427638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767201.195670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767201.451628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767202.219662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767202.475575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767203.243727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767203.499557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767204.267774:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767204.523574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767204.715659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767204.715670:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767204.715675:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767204.715696:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2933 00000100:00080000:2.0:1775767204.715707:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767204.715717:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767204.715772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861460352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767204.715819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767204.715823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767204.715845:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767204.715852:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2933 00000100:00080000:2.0:1775767204.715857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767204.715861:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767204.715873:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767204.715877:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861460352:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767204.715896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861460480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767204.715918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767204.715923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767204.715942:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767204.715947:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2933 00000100:00080000:2.0:1775767204.715953:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767204.715957:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767204.715963:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767204.715976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861460608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767204.715983:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767204.716078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767204/real 1775767204] req@ffffa0e9d0711c00 x1862023861460352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767204.716103:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861460352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767204.716116:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767204.716122:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767204.716126:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861460352:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767204.716153:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767204.716175:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767204.716195:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767204/real 1775767204] req@ffffa0e9d0713b80 x1862023861460480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767204.716204:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861460480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767204.716227:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767204.716230:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767204.716233:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460480:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767204.716238:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767204.716254:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767204.716261:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767204/real 1775767204] req@ffffa0e9d0710000 x1862023861460608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767204.716269:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861460608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767204.716275:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767204.716278:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767204.716280:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460608:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767205.291696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767205.547545:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767206.315668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767206.571601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767207.339671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767207.595575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767208.363653:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767208.619629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767209.387639:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767209.643563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767209.835611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767209.835624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767209.835629:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767209.835663:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2938 00000100:00080000:0.0:1775767209.835672:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767209.835679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767209.835713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861460736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767209.835738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767209.835747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767209.835749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767209.835752:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2938 00000100:00080000:0.0:1775767209.835754:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767209.835758:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767209.835767:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767209.835776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861460864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767209.835782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767209.835785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767209.835786:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767209.835788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2938 00000100:00080000:0.0:1775767209.835790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767209.835792:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767209.835795:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767209.835801:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861460992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767209.835950:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861460736:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767209.836109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767209.836184:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767209/real 1775767209] req@ffffa0e9d09df480 x1862023861460736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767209.836211:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861460736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767209.836265:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767209.836272:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767209.836277:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861460736:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767209.836287:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767209.836318:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767209.836334:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767209/real 1775767209] req@ffffa0e9d09dea00 x1862023861460864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767209.836349:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861460864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767209.836359:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767209.836363:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767209.836367:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460864:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767209.836376:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767209.836411:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767209.836433:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767209/real 1775767209] req@ffffa0e9d09de300 x1862023861460992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767209.836457:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861460992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767209.836474:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767209.836481:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767209.836544:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861460992:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767210.411661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767210.667557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767211.435656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767211.691556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767212.459705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767212.715557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767213.483669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767213.739561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767214.507641:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767214.763635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767214.955633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767214.955642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767214.955647:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767214.955658:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2943 00000100:00080000:1.0:1775767214.955670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767214.955677:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767214.955721:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861461120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767214.955747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767214.955750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767214.955752:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767214.955755:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2943 00000100:00080000:1.0:1775767214.955758:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767214.955761:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767214.955772:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767214.955782:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861461120:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767214.955783:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861461248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767214.955797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767214.955799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767214.955801:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767214.955819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2943 00000100:00080000:1.0:1775767214.955823:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767214.955826:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767214.955831:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767214.955840:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861461376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767214.955857:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767214.955898:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767214/real 1775767214] req@ffffa0e9c8dece00 x1862023861461120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767214.955925:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861461120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767214.955934:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767214.955937:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767214.955940:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861461120:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767214.955961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767214.955977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767214.955985:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767214/real 1775767214] req@ffffa0e9c8dedf80 x1862023861461248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767214.955993:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861461248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767214.955998:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767214.956000:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767214.956002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461248:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767214.956006:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767214.956028:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767214.956035:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767214/real 1775767214] req@ffffa0e9c8deca80 x1862023861461376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767214.956048:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861461376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767214.956054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767214.956056:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767214.956058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461376:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767215.531648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767215.787565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767216.555660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767216.811609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767217.579705:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767217.835610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767218.603712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767218.859605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767219.627664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767219.883658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767220.075558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767220.075568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767220.075573:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767220.075581:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2948 00000100:00080000:1.0:1775767220.075591:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767220.075597:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767220.075633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861461504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767220.075657:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767220.075660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767220.075661:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767220.075664:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2948 00000100:00080000:1.0:1775767220.075666:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767220.075669:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767220.075678:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767220.075705:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861461504:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767220.075708:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861461632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767220.075718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767220.075720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767220.075721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767220.075724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2948 00000100:00080000:1.0:1775767220.075727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767220.075730:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767220.075734:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767220.075742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861461760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767220.075775:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767220.075816:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767220/real 1775767220] req@ffffa0e9c8ded880 x1862023861461504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767220.075832:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861461504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767220.075840:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767220.075844:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767220.075848:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861461504:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767220.075888:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767220.075912:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767220.075924:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767220/real 1775767220] req@ffffa0e9c8dee680 x1862023861461632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767220.075940:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861461632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767220.075945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767220.075948:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767220.075950:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461632:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767220.075954:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767220.075968:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767220.075976:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767220/real 1775767220] req@ffffa0e9c8deea00 x1862023861461760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767220.075988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861461760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767220.075996:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767220.075999:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767220.076002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861461760:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767220.651646:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767220.907591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767221.675713:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767221.931609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767222.699665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767222.955608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767223.723695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767223.979546:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767224.747688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767225.003610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767225.195577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767225.195585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767225.195590:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767225.195598:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2953 00000100:00080000:1.0:1775767225.195607:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767225.195613:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767225.195650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861461888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767225.195675:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767225.195678:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767225.195697:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767225.195700:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2953 00000100:00080000:1.0:1775767225.195703:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767225.195706:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767225.195715:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767225.195723:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861462016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767225.195730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767225.195733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767225.195734:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767225.195737:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2953 00000100:00080000:1.0:1775767225.195739:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767225.195742:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767225.195745:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767225.195750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861462144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767225.195798:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861461888:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767225.195942:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462016:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767225.195967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462144:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767225.771695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767226.027646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767226.795652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767227.051585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767227.819737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767228.075585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775767228.267835:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775767228.267891:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767228.267910:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767228.267961:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767228.267972:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767228.268077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767225/real 1775767228] req@ffffa0e9c8dedc00 x1862023861461888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767228.268113:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861461888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767228.268127:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767228.268133:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767228.268138:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861461888:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767228.268147:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767225/real 1775767228] req@ffffa0e9c8def480 x1862023861462016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767228.268162:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861462016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767228.268172:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767228.268176:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767228.268180:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462016:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767228.268188:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767225/real 1775767228] req@ffffa0e9c8def800 x1862023861462144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767228.268202:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861462144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767228.268210:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767228.268214:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767228.268216:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462144:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767228.843706:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767229.099590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767229.867697:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767230.123623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767230.315605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767230.315615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767230.315620:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767230.315630:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2958 00000100:00080000:1.0:1775767230.315643:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767230.315666:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767230.315705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861462272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767230.315734:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767230.315737:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767230.315738:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767230.315741:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2958 00000100:00080000:1.0:1775767230.315744:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767230.315746:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767230.315756:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767230.315774:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861462400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767230.315782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767230.315785:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767230.315788:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767230.315793:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2958 00000100:00080000:1.0:1775767230.315797:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767230.315814:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767230.315820:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767230.315823:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861462272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767230.315837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861462528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767230.315915:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767230.315963:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767230/real 1775767230] req@ffffa0e9c8dec380 x1862023861462272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767230.315985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861462272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767230.315997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767230.316002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767230.316006:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861462272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767230.316015:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767230.316038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767230.316052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767230/real 1775767230] req@ffffa0e9c8ded500 x1862023861462400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767230.316065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861462400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767230.316073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767230.316076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767230.316080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462400:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767230.316108:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767230.316127:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767230.316138:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767230/real 1775767230] req@ffffa0e9c8deed80 x1862023861462528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767230.316165:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861462528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767230.316173:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767230.316176:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767230.316179:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462528:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767230.891650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767231.147605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767231.915713:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767232.171672:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767232.939704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767233.195567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767233.963648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767234.219614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767234.987695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767235.243566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767235.435598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767235.435607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767235.435611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767235.435619:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2963 00000100:00080000:2.0:1775767235.435629:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767235.435635:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767235.435697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861462656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767235.435732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767235.435735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767235.435737:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767235.435740:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2963 00000100:00080000:2.0:1775767235.435743:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767235.435746:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767235.435755:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767235.435766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861462784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767235.435774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767235.435777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767235.435780:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767235.435782:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861462656:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767235.435787:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2963 00000100:00080000:2.0:1775767235.435792:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767235.435796:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767235.435811:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767235.435827:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861462912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767235.435850:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767235.435901:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767235/real 1775767235] req@ffffa0e9d0711180 x1862023861462656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767235.435921:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861462656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767235.435933:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767235.435937:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767235.435941:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861462656:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767235.435966:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767235.435986:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767235.435997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767235/real 1775767235] req@ffffa0e9d0710380 x1862023861462784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767235.436009:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861462784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767235.436024:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767235.436028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767235.436031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462784:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767235.436037:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767235.436055:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767235.436064:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767235/real 1775767235] req@ffffa0e9d0710000 x1862023861462912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767235.436076:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861462912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767235.436084:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767235.436087:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767235.436090:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861462912:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767236.011630:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767236.267622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767237.035741:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767237.291594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767238.059700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767238.315624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767239.083736:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767239.339623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767240.107700:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767240.363608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767240.555638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767240.555649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767240.555663:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767240.555686:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2969 00000100:00080000:1.0:1775767240.555698:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767240.555705:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767240.555748:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861463040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767240.555778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767240.555782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767240.555784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767240.555788:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2969 00000100:00080000:1.0:1775767240.555792:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767240.555796:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767240.555808:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767240.555828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861463168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767240.555839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767240.555842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767240.555843:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767240.555850:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2969 00000100:00100000:0.0:1775767240.555857:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861463040:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767240.555871:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767240.555875:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767240.555886:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767240.555900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861463296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767240.555942:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767240.555988:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767240/real 1775767240] req@ffffa0e9c8dee300 x1862023861463040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767240.556005:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861463040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767240.556014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767240.556029:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767240.556033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861463040:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767240.556056:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767240.556074:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767240.556083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767240/real 1775767240] req@ffffa0e9c8dec000 x1862023861463168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767240.556092:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861463168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767240.556097:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767240.556100:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767240.556102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463168:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767240.556106:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767240.556117:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767240.556123:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767240/real 1775767240] req@ffffa0e9c8ded180 x1862023861463296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767240.556131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861463296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767240.556136:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767240.556138:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767240.556139:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463296:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767241.131715:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767241.387604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767242.155714:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767242.411601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767243.179708:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767243.435590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767244.203659:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767244.459610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767245.227679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767245.483589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767245.675584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767245.675594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767245.675598:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767245.675613:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2974 00000100:00080000:2.0:1775767245.675636:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767245.675643:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767245.675687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861463424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767245.675716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767245.675719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767245.675720:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767245.675723:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2974 00000100:00080000:2.0:1775767245.675727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767245.675730:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767245.675738:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767245.675749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861463552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767245.675758:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767245.675760:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767245.675768:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861463424:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767245.675780:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767245.675784:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2974 00000100:00080000:2.0:1775767245.675787:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767245.675790:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767245.675794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767245.675802:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861463680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767245.675836:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767245.675881:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767245/real 1775767245] req@ffffa0e9d0713b80 x1862023861463424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767245.675895:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861463424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767245.675903:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767245.675906:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767245.675909:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861463424:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767245.675928:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767245.675948:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767245.675960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767245/real 1775767245] req@ffffa0e9d0711c00 x1862023861463552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767245.675973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861463552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767245.675982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767245.675986:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767245.675990:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463552:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767245.675996:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767245.676015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767245.676023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767245/real 1775767245] req@ffffa0e9d0711f80 x1862023861463680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767245.676044:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861463680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767245.676049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767245.676052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767245.676054:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463680:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767246.251688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767246.507583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767247.275742:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767247.531625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767248.299645:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767248.555676:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767249.323740:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767249.579609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767250.347619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767250.603597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767250.795633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767250.795643:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767250.795647:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767250.795657:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2979 00000100:00080000:2.0:1775767250.795667:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767250.795673:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767250.795709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861463808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767250.795740:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767250.795743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767250.795745:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767250.795749:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2979 00000100:00080000:2.0:1775767250.795770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767250.795774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767250.795783:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767250.795789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861463808:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767250.795796:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861463936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767250.795805:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767250.795808:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767250.795810:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767250.795814:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2979 00000100:00080000:2.0:1775767250.795819:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767250.795823:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767250.795828:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767250.795839:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861464064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767250.795849:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767250.795888:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767250/real 1775767250] req@ffffa0e9d0710700 x1862023861463808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767250.795902:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861463808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767250.795909:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767250.795913:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767250.795916:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861463808:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767250.795934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767250.795955:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767250.795969:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767250/real 1775767250] req@ffffa0e9d0712d80 x1862023861463936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767250.795995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861463936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767250.796003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767250.796006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767250.796009:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861463936:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767250.796015:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767250.796036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767250.796047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767250/real 1775767250] req@ffffa0e9d0712680 x1862023861464064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767250.796059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861464064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767250.796065:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767250.796068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767250.796071:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464064:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767251.371697:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767251.627601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767252.395695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767252.651599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767253.419761:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767253.675609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767254.443686:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767254.699609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767255.467695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767255.723635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767255.915664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767255.915676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767255.915682:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767255.915694:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2984 00000100:00080000:1.0:1775767255.915708:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767255.915716:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767255.915784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861464192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767255.915813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767255.915816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767255.915818:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767255.915821:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2984 00000100:00080000:1.0:1775767255.915825:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767255.915828:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767255.915844:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767255.915855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861464320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767255.915864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767255.915867:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767255.915869:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767255.915874:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2984 00000100:00100000:0.0:1775767255.915877:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861464192:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767255.915882:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767255.915887:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767255.915891:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767255.915900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861464448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767255.915962:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767255.916023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767255/real 1775767255] req@ffffa0e9c8ded180 x1862023861464192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767255.916046:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861464192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767255.916058:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767255.916063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767255.916069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861464192:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767255.916097:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767255.916120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767255.916134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767255/real 1775767255] req@ffffa0e9c8dec000 x1862023861464320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767255.916162:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861464320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767255.916173:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767255.916176:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767255.916180:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464320:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767255.916187:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767255.916210:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767255.916221:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767255/real 1775767255] req@ffffa0e9c8dee300 x1862023861464448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767255.916235:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861464448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767255.916245:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767255.916248:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767255.916252:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464448:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767256.491671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767256.747594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767257.515722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767257.771617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767258.539680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767258.795628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767259.563660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767259.819624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767260.587646:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767260.843605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767261.035596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767261.035628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767261.035634:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767261.035646:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2989 00000100:00080000:1.0:1775767261.035662:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767261.035671:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767261.035714:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861464576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767261.035749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767261.035753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767261.035755:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767261.035760:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2989 00000100:00080000:1.0:1775767261.035764:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767261.035768:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767261.035781:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767261.035796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861464704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767261.035809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767261.035814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767261.035816:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767261.035821:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861464576:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767261.035824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2989 00000100:00080000:1.0:1775767261.035833:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767261.035838:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767261.035844:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767261.035858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861464832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767261.035912:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767261.035962:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767261/real 1775767261] req@ffffa0e9c8deed80 x1862023861464576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767261.035984:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861464576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767261.036004:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767261.036009:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767261.036014:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861464576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767261.036044:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767261.036070:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767261.036085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767261/real 1775767261] req@ffffa0e9c8ded500 x1862023861464704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767261.036098:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861464704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767261.036107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767261.036111:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767261.036114:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464704:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767261.036121:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767261.036141:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767261.036152:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767261/real 1775767261] req@ffffa0e9c8dec380 x1862023861464832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767261.036165:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861464832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767261.036174:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767261.036177:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767261.036181:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861464832:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767261.611673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767261.867623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767262.635653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767262.891608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767263.659640:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767263.915628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767264.683726:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767264.939609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767265.707656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767265.963605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767266.155567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767266.155576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767266.155582:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767266.155595:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2994 00000100:00080000:1.0:1775767266.155609:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767266.155617:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767266.155666:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861464960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767266.155694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767266.155698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767266.155699:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767266.155702:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2994 00000100:00080000:1.0:1775767266.155706:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767266.155708:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767266.155718:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767266.155728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861465088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767266.155736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767266.155739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767266.155742:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767266.155744:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861464960:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767266.155748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2994 00000100:00080000:1.0:1775767266.155755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767266.155778:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767266.155785:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767266.155800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861465216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767266.155820:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767266.155868:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767266/real 1775767266] req@ffffa0e9c8def800 x1862023861464960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767266.155886:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861464960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767266.155895:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767266.155898:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767266.155901:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861464960:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767266.155921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767266.155936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767266.155945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767266/real 1775767266] req@ffffa0e9c8def480 x1862023861465088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767266.155952:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861465088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767266.155958:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767266.155960:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767266.155963:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767266.155966:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767266.155989:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767266.155996:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767266/real 1775767266] req@ffffa0e9c8dedc00 x1862023861465216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767266.156004:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861465216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767266.156009:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767266.156012:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767266.156013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465216:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767266.731649:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767266.987578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767267.755691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767268.011638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767268.779677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767269.035566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767269.803666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767270.059609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767270.827691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767271.083609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767271.275638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767271.275647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767271.275653:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767271.275663:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 2999 00000100:00080000:2.0:1775767271.275674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767271.275681:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767271.275724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861465344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767271.275754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767271.275758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767271.275759:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767271.275762:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2999 00000100:00080000:2.0:1775767271.275765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767271.275767:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767271.275792:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767271.275803:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861465472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767271.275815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767271.275820:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861465344:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767271.275820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767271.275825:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767271.275833:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 2999 00000100:00080000:2.0:1775767271.275838:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767271.275842:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767271.275848:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767271.275863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861465600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767271.275897:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767271.275939:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767271/real 1775767271] req@ffffa0e9d0710a80 x1862023861465344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767271.275954:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861465344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767271.275963:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767271.275966:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767271.275969:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861465344:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767271.275989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767271.276003:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767271.276011:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767271/real 1775767271] req@ffffa0e9d0710e00 x1862023861465472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767271.276028:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861465472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767271.276034:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767271.276036:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767271.276038:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465472:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767271.276042:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767271.276055:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767271.276062:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767271/real 1775767271] req@ffffa0e9d0712a00 x1862023861465600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767271.276070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861465600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767271.276074:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767271.276076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767271.276078:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465600:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767271.851666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767272.107612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767272.875730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767273.131616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767273.899690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767274.155619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767274.923717:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767275.179628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767275.947684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767276.203611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767276.395612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767276.395625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767276.395631:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767276.395643:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3004 00000100:00080000:0.0:1775767276.395657:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767276.395666:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767276.395725:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861465728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767276.395758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767276.395763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767276.395765:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767276.395769:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3004 00000100:00080000:0.0:1775767276.395773:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767276.395776:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767276.395786:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767276.395796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861465856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767276.395813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767276.395816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767276.395818:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767276.395823:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861465728:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767276.395824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3004 00000100:00080000:0.0:1775767276.395829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767276.395834:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767276.395837:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767276.395845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861465984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767276.395909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767276.395953:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767276/real 1775767276] req@ffffa0e9d09dc000 x1862023861465728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767276.395969:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861465728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767276.395978:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767276.395981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767276.395984:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861465728:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767276.396005:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767276.396022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767276.396030:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767276/real 1775767276] req@ffffa0e9d09dc380 x1862023861465856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767276.396038:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861465856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767276.396054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767276.396057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767276.396059:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465856:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767276.396063:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767276.396078:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767276.396084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767276/real 1775767276] req@ffffa0e9d09dce00 x1862023861465984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767276.396092:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861465984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767276.396097:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767276.396099:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767276.396101:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861465984:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767276.971680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767277.227614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767277.995677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767278.251728:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767279.019643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767279.275625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767280.043679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767280.299598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767281.067721:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767281.323615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767281.515628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767281.515636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767281.515642:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767281.515652:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3010 00000100:00080000:3.0:1775767281.515664:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767281.515670:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767281.515712:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861466112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767281.515739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767281.515742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767281.515743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767281.515746:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3010 00000100:00080000:3.0:1775767281.515748:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767281.515751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767281.515761:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767281.515769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861466240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767281.515777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767281.515780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767281.515781:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767281.515784:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3010 00000100:00080000:3.0:1775767281.515786:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767281.515790:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767281.515793:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861466112:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775767281.515810:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767281.515821:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861466368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767281.515868:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767281.515913:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767281/real 1775767281] req@ffffa0e9d0541180 x1862023861466112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767281.515932:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861466112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767281.515941:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767281.515945:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767281.515954:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861466112:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767281.515960:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767281.515977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767281.515986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767281/real 1775767281] req@ffffa0e9d0543480 x1862023861466240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767281.515994:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861466240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767281.516000:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767281.516002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767281.516005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466240:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767281.516027:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767281.516037:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767281.516043:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767281/real 1775767281] req@ffffa0e9d0543100 x1862023861466368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767281.516050:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861466368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767281.516055:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767281.516057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767281.516059:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466368:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767282.091674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767282.347678:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767283.115717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767283.371606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767284.139670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767284.395589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767285.163694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767285.419604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767286.187680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767286.443569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767286.635639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767286.635652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767286.635657:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767286.635670:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3015 00000100:00080000:3.0:1775767286.635682:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767286.635689:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767286.635731:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861466496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767286.635766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767286.635770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767286.635773:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767286.635777:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3015 00000100:00080000:3.0:1775767286.635781:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767286.635785:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767286.635797:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767286.635812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861466624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767286.635825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767286.635828:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861466496:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767286.635829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767286.635834:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767286.635842:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3015 00000100:00080000:3.0:1775767286.635866:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767286.635871:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767286.635877:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767286.635890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861466752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767286.635909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767286.635968:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767286/real 1775767286] req@ffffa0e9d0543100 x1862023861466496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767286.635992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861466496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767286.636003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767286.636007:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767286.636012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861466496:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767286.636037:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767286.636063:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767286.636077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767286/real 1775767286] req@ffffa0e9d0543480 x1862023861466624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767286.636091:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861466624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767286.636100:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767286.636104:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767286.636107:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466624:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767286.636114:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767286.636136:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767286.636148:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767286/real 1775767286] req@ffffa0e9d0541180 x1862023861466752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767286.636179:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861466752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767286.636188:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767286.636192:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767286.636196:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861466752:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767287.211655:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767287.467612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767288.235695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767288.491590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767289.259631:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767289.515600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767290.283682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767290.539602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767291.307610:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767291.563577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767291.755639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767291.755649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767291.755656:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767291.755668:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3020 00000100:00080000:3.0:1775767291.755682:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767291.755691:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767291.755738:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861466880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767291.755771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767291.755775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767291.755777:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767291.755782:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3020 00000100:00080000:3.0:1775767291.755786:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767291.755790:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767291.755801:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767291.755814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861467008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767291.755823:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861466880:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767291.755830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767291.755850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767291.755852:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767291.755858:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3020 00000100:00080000:3.0:1775767291.755863:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767291.755868:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767291.755874:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767291.755887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861467136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767291.755948:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467008:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767291.755985:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467136:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767292.331802:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767292.587611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767293.355710:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767293.611596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767294.379704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767294.635606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:1.0:1775767294.827787:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:1.0:1775767294.827829:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767294.827847:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775767294.827887:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775767294.827895:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767294.827956:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767291/real 1775767294] req@ffffa0e9d0541f80 x1862023861467008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.827990:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861467008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.828003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.828009:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767294.828013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467008:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767294.828022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767291/real 1775767294] req@ffffa0e9d0542d80 x1862023861467136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.828034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861467136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.828069:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.828074:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767294.828078:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467136:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767294.828085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767291/real 1775767294] req@ffffa0e9d0540a80 x1862023861466880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.828096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861466880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.828104:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.828108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767294.828111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861466880:192.168.202.104@tcp:250:kworker.0 00010000:00010000:0.0:1775767295.403649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767295.659601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767296.427673:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767296.683633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767296.875652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767296.875660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767296.875665:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767296.875675:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3025 00000100:00080000:1.0:1775767296.875687:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767296.875693:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767296.875730:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861467264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767296.875755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767296.875758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767296.875760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767296.875763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3025 00000100:00080000:1.0:1775767296.875765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767296.875768:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767296.875777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767296.875786:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861467392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767296.875808:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861467264:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767296.875818:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767296.875822:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767296.875823:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767296.875827:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3025 00000100:00080000:1.0:1775767296.875830:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767296.875833:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767296.875837:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767296.875847:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861467520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767296.875903:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767296.875951:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767296/real 1775767296] req@ffffa0e9c8deea00 x1862023861467264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767296.875973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861467264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767296.875987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767296.875992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767296.875996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861467264:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767296.876026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767296.876073:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767296.876089:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767296/real 1775767296] req@ffffa0e9c8dee680 x1862023861467392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767296.876111:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861467392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767296.876120:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767296.876124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767296.876127:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467392:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767296.876134:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767296.876171:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767296.876184:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767296/real 1775767296] req@ffffa0e9c8ded880 x1862023861467520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767296.876196:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861467520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767296.876204:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767296.876209:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767296.876212:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467520:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767297.451677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767297.707656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767298.475682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767298.731610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767299.499674:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767299.755604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767300.523701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767300.779597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767301.547653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767301.803592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767301.995572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767301.995585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767301.995591:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767301.995604:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3030 00000100:00080000:3.0:1775767301.995618:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767301.995626:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767301.995670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861467648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767301.995701:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767301.995705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767301.995708:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767301.995712:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3030 00000100:00080000:3.0:1775767301.995716:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767301.995721:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767301.995731:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767301.995748:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861467648:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767301.995748:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861467776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767301.995773:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767301.995777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767301.995779:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767301.995784:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3030 00000100:00080000:3.0:1775767301.995789:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767301.995802:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767301.995811:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767301.995824:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861467904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767301.995834:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767301.995885:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767301/real 1775767301] req@ffffa0e9d0540e00 x1862023861467648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767301.995919:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861467648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767301.995932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767301.995938:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767301.995943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861467648:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767301.995972:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767301.996000:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767301.996014:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767301/real 1775767301] req@ffffa0e9d0540700 x1862023861467776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767301.996039:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861467776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767301.996049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767301.996053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767301.996057:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467776:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767301.996071:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767301.996108:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767301.996122:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767301/real 1775767301] req@ffffa0e9d0543800 x1862023861467904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767301.996136:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861467904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767301.996159:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767301.996163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767301.996167:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861467904:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767302.571683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767302.827623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767303.595637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767303.851617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767304.619680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767304.875610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767305.643656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767305.899632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767306.667697:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767306.923636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767307.115593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767307.115606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767307.115611:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767307.115622:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3035 00000100:00080000:3.0:1775767307.115633:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767307.115639:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767307.115676:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861468032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767307.115703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767307.115706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767307.115707:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767307.115710:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3035 00000100:00080000:3.0:1775767307.115712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767307.115715:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767307.115741:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767307.115751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861468160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767307.115758:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861468032:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767307.115762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767307.115765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767307.115766:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767307.115769:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3035 00000100:00080000:3.0:1775767307.115772:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767307.115775:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767307.115779:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767307.115786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861468288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767307.115820:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767307.115858:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767307/real 1775767307] req@ffffa0e9d0541c00 x1862023861468032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767307.115872:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861468032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767307.115880:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767307.115883:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767307.115886:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861468032:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767307.115906:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767307.115919:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767307.115927:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767307/real 1775767307] req@ffffa0e9d0542300 x1862023861468160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767307.115935:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861468160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767307.115940:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767307.115942:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767307.115951:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468160:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767307.115956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767307.115969:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767307.115975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767307/real 1775767307] req@ffffa0e9d0540380 x1862023861468288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767307.115982:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861468288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767307.115987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767307.115989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767307.115991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468288:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767307.691711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767307.947567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767308.715643:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767308.971614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767309.739720:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767309.995604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767310.763640:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767311.019593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767311.787701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767312.043654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767312.235608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767312.235617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767312.235621:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767312.235632:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3040 00000100:00080000:3.0:1775767312.235644:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767312.235652:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767312.235693:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861468416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767312.235719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767312.235722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767312.235724:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767312.235726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3040 00000100:00080000:3.0:1775767312.235747:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767312.235752:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767312.235761:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767312.235763:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861468416:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767312.235783:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861468544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767312.235792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767312.235795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767312.235796:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767312.235800:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3040 00000100:00080000:3.0:1775767312.235802:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767312.235804:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767312.235807:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767312.235817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861468672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767312.235824:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767312.235864:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767312/real 1775767312] req@ffffa0e9d0540000 x1862023861468416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767312.235879:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861468416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767312.235888:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767312.235891:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767312.235894:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861468416:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767312.235912:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767312.235936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767312.235945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767312/real 1775767312] req@ffffa0e9d0542a00 x1862023861468544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767312.235953:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861468544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767312.235959:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767312.235961:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767312.235963:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468544:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767312.235967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767312.235977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767312.235982:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767312/real 1775767312] req@ffffa0e9d0543b80 x1862023861468672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767312.235989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861468672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767312.235994:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767312.236011:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767312.236015:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468672:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767312.811694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767313.067616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767313.835686:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767314.091649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767314.859644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767315.115591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767315.883680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767316.139602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767316.907677:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767317.163629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767317.355621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767317.355630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767317.355634:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767317.355642:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3045 00000100:00080000:0.0:1775767317.355673:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767317.355678:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767317.355710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861468800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767317.355736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767317.355739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767317.355740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767317.355743:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3045 00000100:00080000:0.0:1775767317.355745:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767317.355748:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767317.355756:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767317.355765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861468928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767317.355773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767317.355777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767317.355779:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767317.355779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861468800:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767317.355786:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3045 00000100:00080000:0.0:1775767317.355789:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767317.355791:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767317.355794:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767317.355804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861469056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767317.355846:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767317.355885:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767317/real 1775767317] req@ffffa0e9d09df100 x1862023861468800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767317.355906:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861468800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767317.355918:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767317.355922:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767317.355934:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861468800:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767317.355991:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767317.356011:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767317.356021:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767317/real 1775767317] req@ffffa0e9d09dfb80 x1862023861468928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767317.356030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861468928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767317.356035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767317.356037:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767317.356040:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861468928:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767317.356043:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767317.356054:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767317.356059:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767317/real 1775767317] req@ffffa0e9d09ded80 x1862023861469056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767317.356067:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861469056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767317.356072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767317.356073:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767317.356075:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469056:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767317.931706:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767318.187621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767318.955672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767319.211587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767319.979675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767320.235642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767321.003627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767321.259599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767322.027679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767322.283636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767322.475621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767322.475629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767322.475633:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767322.475643:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3050 00000100:00080000:3.0:1775767322.475652:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767322.475657:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767322.475696:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861469184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767322.475719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767322.475740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767322.475742:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767322.475745:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3050 00000100:00080000:3.0:1775767322.475748:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767322.475752:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767322.475760:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767322.475771:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861469312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767322.475779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767322.475781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767322.475783:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767322.475786:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3050 00000100:00080000:3.0:1775767322.475788:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767322.475791:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767322.475795:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767322.475805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861469184:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767322.475806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861469440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767322.475897:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767322.475954:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767322/real 1775767322] req@ffffa0e9d0542680 x1862023861469184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767322.475971:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861469184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767322.475981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767322.475984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767322.475988:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861469184:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767322.475993:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767322.476010:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767322.476018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767322/real 1775767322] req@ffffa0e9d0541500 x1862023861469312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767322.476027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861469312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767322.476033:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767322.476035:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767322.476038:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469312:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767322.476061:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767322.476071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767322.476078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767322/real 1775767322] req@ffffa0e9d0541880 x1862023861469440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767322.476104:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861469440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767322.476110:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767322.476112:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767322.476114:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469440:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767323.051614:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767323.307616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767324.075667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767324.331602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767325.099725:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767325.355579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767326.123682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767326.379602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767327.147699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767327.403554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767327.595575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767327.595587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767327.595592:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767327.595603:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3056 00000100:00080000:3.0:1775767327.595615:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767327.595621:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767327.595670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861469568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767327.595714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767327.595717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767327.595718:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767327.595722:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3056 00000100:00080000:3.0:1775767327.595725:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767327.595729:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767327.595739:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767327.595765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861469696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767327.595775:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861469568:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767327.595780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767327.595785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767327.595787:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767327.595793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3056 00000100:00080000:3.0:1775767327.595798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767327.595803:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767327.595811:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767327.595835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861469824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767327.595873:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767327.595938:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767327/real 1775767327] req@ffffa0e9d0541880 x1862023861469568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767327.595956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861469568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767327.595966:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767327.595970:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767327.595973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861469568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767327.595997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767327.596015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767327.596024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767327/real 1775767327] req@ffffa0e9d0541500 x1862023861469696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767327.596032:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861469696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767327.596048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767327.596051:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767327.596053:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469696:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767327.596057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767327.596071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767327.596077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767327/real 1775767327] req@ffffa0e9d0542680 x1862023861469824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767327.596085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861469824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767327.596090:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767327.596092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767327.596094:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861469824:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767328.171655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767328.427620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767329.195727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767329.451613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767330.219681:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767330.475594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767331.243694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767331.499643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767332.267678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767332.523601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767332.715670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767332.715696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767332.715701:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767332.715710:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3061 00000100:00080000:0.0:1775767332.715721:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767332.715727:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767332.715770:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861469952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767332.715828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767332.715832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767332.715835:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767332.715840:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3061 00000100:00080000:0.0:1775767332.715845:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767332.715851:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767332.715863:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767332.715875:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861469952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767332.715881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861470080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767332.715896:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767332.715900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767332.715903:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767332.715908:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3061 00000100:00080000:0.0:1775767332.715912:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767332.715917:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767332.715923:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767332.715935:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861470208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767332.715965:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767332.716009:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767332/real 1775767332] req@ffffa0e9d09dc700 x1862023861469952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767332.716026:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861469952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767332.716034:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767332.716043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767332.716047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861469952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767332.716069:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767332.716088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767332.716097:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767332/real 1775767332] req@ffffa0e9d09df800 x1862023861470080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767332.716106:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861470080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767332.716111:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767332.716114:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767332.716116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470080:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767332.716120:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767332.716135:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767332.716144:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767332/real 1775767332] req@ffffa0e9d09dd880 x1862023861470208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767332.716156:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861470208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767332.716165:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767332.716169:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767332.716173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470208:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767333.291710:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767333.547613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767334.315740:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767334.571580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767335.339711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767335.595592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767336.363660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767336.619594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767337.387728:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767337.643602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767337.835587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767337.835595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767337.835600:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767337.835610:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3066 00000100:00080000:0.0:1775767337.835622:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767337.835628:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767337.835671:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861470336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767337.835698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767337.835701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767337.835702:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767337.835705:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3066 00000100:00080000:0.0:1775767337.835708:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767337.835710:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767337.835719:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767337.835728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861470464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767337.835736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767337.835739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767337.835740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767337.835743:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3066 00000100:00080000:0.0:1775767337.835744:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767337.835746:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767337.835782:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767337.835791:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861470336:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767337.835793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861470592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767337.835877:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767337.835933:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767337/real 1775767337] req@ffffa0e9d09dd500 x1862023861470336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767337.835957:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861470336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767337.835969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767337.835973:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767337.835978:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861470336:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767337.835986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767337.836009:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767337.836022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767337/real 1775767337] req@ffffa0e9d09dd180 x1862023861470464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767337.836035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861470464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767337.836045:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767337.836049:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767337.836052:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470464:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767337.836078:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767337.836098:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767337.836108:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767337/real 1775767337] req@ffffa0e9d09ddc00 x1862023861470592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767337.836130:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861470592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767337.836139:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767337.836143:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767337.836147:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470592:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767338.411619:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767338.667589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767339.435725:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767339.691616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767340.459666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767340.715617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767341.483759:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767341.739604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767342.507758:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767342.763586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767342.955601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767342.955611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767342.955616:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767342.955627:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3071 00000100:00080000:0.0:1775767342.955639:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767342.955646:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767342.955688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861470720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767342.955731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767342.955734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767342.955736:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767342.955740:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3071 00000100:00080000:0.0:1775767342.955745:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767342.955749:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767342.955759:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767342.955770:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861470848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767342.955778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767342.955782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767342.955783:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767342.955786:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3071 00000100:00100000:1.0:1775767342.955802:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861470720:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767342.955805:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767342.955816:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767342.955821:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767342.955833:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861470976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767342.955887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767342.955939:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767342/real 1775767342] req@ffffa0e9d09ddf80 x1862023861470720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767342.955961:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861470720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767342.955970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767342.955974:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767342.955977:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861470720:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767342.955984:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767342.956001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767342.956009:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767342/real 1775767342] req@ffffa0e9d09dca80 x1862023861470848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767342.956017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861470848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767342.956040:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767342.956043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767342.956045:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470848:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767342.956065:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767342.956080:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767342.956086:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767342/real 1775767342] req@ffffa0e9d09de680 x1862023861470976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767342.956094:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861470976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767342.956099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767342.956100:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767342.956102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861470976:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767343.531691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767343.787654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767344.555689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767344.811587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767345.579675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767345.835656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767346.603664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767346.859580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767347.627664:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767347.883572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767348.075597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767348.075609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767348.075615:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767348.075627:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3076 00000100:00080000:0.0:1775767348.075640:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767348.075646:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767348.075709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861471104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767348.075741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767348.075745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767348.075747:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767348.075751:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3076 00000100:00080000:0.0:1775767348.075755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767348.075759:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767348.075771:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767348.075786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861471232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767348.075797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767348.075816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767348.075819:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767348.075825:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3076 00000100:00080000:0.0:1775767348.075829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767348.075834:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767348.075841:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767348.075846:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861471104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767348.075854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861471360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767348.075947:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767348.076015:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767348/real 1775767348] req@ffffa0e9d09de680 x1862023861471104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767348.076038:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861471104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767348.076047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767348.076062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767348.076066:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861471104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767348.076073:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767348.076091:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767348.076101:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767348/real 1775767348] req@ffffa0e9d09dca80 x1862023861471232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767348.076109:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861471232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767348.076125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767348.076127:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767348.076130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767348.076149:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767348.076160:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767348.076166:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767348/real 1775767348] req@ffffa0e9d09ddf80 x1862023861471360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767348.076173:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861471360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767348.076178:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767348.076180:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767348.076182:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471360:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767348.651718:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767348.907597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767349.675713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767349.931593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767350.699716:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767350.955657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767351.723665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767351.979595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767352.747750:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767353.003649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767353.195668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767353.195682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767353.195688:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767353.195702:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3081 00000100:00080000:3.0:1775767353.195734:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767353.195743:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767353.195804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861471488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767353.195860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767353.195864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767353.195867:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767353.195873:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3081 00000100:00080000:3.0:1775767353.195877:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767353.195882:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767353.195895:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767353.195910:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861471616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767353.195923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767353.195928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767353.195931:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767353.195952:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3081 00000100:00080000:3.0:1775767353.195956:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767353.195960:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767353.195968:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861471488:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775767353.195969:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767353.195987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861471744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767353.196050:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767353.196099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767353/real 1775767353] req@ffffa0e9d0543b80 x1862023861471488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767353.196122:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861471488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767353.196135:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767353.196140:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767353.196148:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861471488:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767353.196154:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767353.196174:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767353.196184:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767353/real 1775767353] req@ffffa0e9d0542a00 x1862023861471616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767353.196193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861471616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767353.196199:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767353.196201:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767353.196204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471616:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767353.196226:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767353.196238:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767353.196244:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767353/real 1775767353] req@ffffa0e9d0540000 x1862023861471744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767353.196260:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861471744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767353.196265:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767353.196268:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767353.196270:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861471744:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767353.771693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767354.027621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767354.795684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767355.051643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767355.819753:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767356.075582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767356.843694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767357.099636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767357.867693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767358.123604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767358.315655:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767358.315684:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767358.315687:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767358.315696:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3086 00000100:00080000:3.0:1775767358.315706:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767358.315712:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767358.315745:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861471872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767358.315773:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767358.315776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767358.315777:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767358.315780:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3086 00000100:00080000:3.0:1775767358.315783:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767358.315786:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767358.315795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767358.315806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861472000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767358.315814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767358.315818:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767358.315819:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767358.315822:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3086 00000100:00080000:3.0:1775767358.315824:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767358.315828:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861471872:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767358.315846:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767358.315854:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767358.315867:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861472128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767358.315975:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472000:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767358.316043:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472128:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767358.891700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767359.147666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767359.915707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767360.171582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767360.939690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767361.195623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775767361.387946:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775767361.388001:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767361.388025:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767361.388099:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767361.388116:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775767361.388221:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767358/real 1775767361] req@ffffa0e9d0541c00 x1862023861472128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767361.388286:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861472128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767361.388305:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767361.388313:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767361.388319:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767361.388347:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767358/real 1775767361] req@ffffa0e9d0540380 x1862023861471872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767361.388365:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861471872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767361.388399:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767361.388406:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767361.388411:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861471872:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767361.388429:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767358/real 1775767361] req@ffffa0e9d0542300 x1862023861472000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767361.388448:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861472000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767361.388460:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767361.388465:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767361.388469:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472000:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767361.963754:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767362.219600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767362.987691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767363.243631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767363.435638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767363.435650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767363.435655:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767363.435664:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3091 00000100:00080000:0.0:1775767363.435673:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767363.435679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767363.435712:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861472256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767363.435735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767363.435738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767363.435739:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767363.435742:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3091 00000100:00080000:0.0:1775767363.435745:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767363.435747:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767363.435755:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767363.435767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861472384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767363.435791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767363.435794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767363.435795:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767363.435798:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3091 00000100:00080000:0.0:1775767363.435801:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767363.435804:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767363.435807:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767363.435814:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861472512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767363.435827:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861472256:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767363.435926:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767363.436007:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767363/real 1775767363] req@ffffa0e9d09ddc00 x1862023861472256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767363.436030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861472256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767363.436043:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767363.436048:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767363.436053:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861472256:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767363.436064:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767363.436109:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767363.436125:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767363/real 1775767363] req@ffffa0e9d09dd180 x1862023861472384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767363.436139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861472384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767363.436149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767363.436153:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767363.436157:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472384:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767363.436194:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767363.436217:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767363.436230:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767363/real 1775767363] req@ffffa0e9d09dd500 x1862023861472512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767363.436243:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861472512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767363.436252:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767363.436256:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767363.436260:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472512:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767364.011735:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767364.267570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767365.035730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767365.291617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767366.059743:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767366.315604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767367.083635:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767367.339605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767368.107698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767368.363690:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767368.555669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767368.555678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767368.555682:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767368.555695:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3097 00000100:00080000:2.0:1775767368.555705:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767368.555711:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767368.555747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861472640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767368.555771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767368.555774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767368.555775:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767368.555778:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3097 00000100:00080000:2.0:1775767368.555780:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767368.555782:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767368.555791:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767368.555798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861472768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767368.555806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767368.555808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767368.555827:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767368.555830:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3097 00000100:00080000:2.0:1775767368.555833:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767368.555837:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767368.555842:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767368.555856:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861472640:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767368.555868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861472896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767368.555936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767368.555980:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767368/real 1775767368] req@ffffa0e9d0713480 x1862023861472640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767368.556011:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861472640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767368.556020:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767368.556024:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767368.556027:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861472640:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767368.556033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767368.556051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767368.556060:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767368/real 1775767368] req@ffffa0e9d0711500 x1862023861472768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767368.556068:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861472768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767368.556074:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767368.556077:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767368.556079:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472768:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767368.556102:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767368.556113:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767368.556119:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767368/real 1775767368] req@ffffa0e9d0711880 x1862023861472896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767368.556142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861472896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767368.556159:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767368.556162:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767368.556165:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861472896:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767369.131655:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767369.387618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767370.155682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767370.411634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767371.179679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767371.435591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767372.203674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767372.459659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767373.227660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767373.483626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767373.675625:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767373.675634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767373.675639:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767373.675648:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3102 00000100:00080000:0.0:1775767373.675657:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767373.675663:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767373.675705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861473024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767373.675735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767373.675739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767373.675741:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767373.675744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3102 00000100:00080000:0.0:1775767373.675747:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767373.675750:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767373.675776:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767373.675789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861473152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767373.675801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767373.675803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767373.675805:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767373.675808:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3102 00000100:00080000:0.0:1775767373.675811:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767373.675820:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861473024:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767373.675821:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767373.675830:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767373.675839:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861473280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767373.675908:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767373.675964:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767373/real 1775767373] req@ffffa0e9d09dd880 x1862023861473024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767373.675982:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861473024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767373.675991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767373.675996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767373.676000:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861473024:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767373.676024:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767373.676041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767373.676050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767373/real 1775767373] req@ffffa0e9d09df800 x1862023861473152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767373.676058:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861473152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767373.676063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767373.676066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767373.676080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473152:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767373.676085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767373.676100:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767373.676107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767373/real 1775767373] req@ffffa0e9d09dc700 x1862023861473280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767373.676115:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861473280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767373.676119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767373.676121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767373.676123:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473280:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767374.251730:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767374.507575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767375.275631:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767375.531550:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767376.299637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767376.555592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767377.323619:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767377.579638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767378.347731:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767378.603548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767378.795554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767378.795564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767378.795570:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767378.795579:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3107 00000100:00080000:0.0:1775767378.795592:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767378.795599:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767378.795647:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861473408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767378.795681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767378.795683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767378.795685:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767378.795688:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3107 00000100:00080000:0.0:1775767378.795706:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767378.795709:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767378.795720:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767378.795731:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861473536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767378.795741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767378.795744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767378.795746:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767378.795751:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3107 00000100:00080000:0.0:1775767378.795757:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767378.795761:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767378.795762:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861473408:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767378.795765:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767378.795777:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861473664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767378.795841:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767378.795881:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767378/real 1775767378] req@ffffa0e9d09ded80 x1862023861473408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767378.795896:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861473408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767378.795905:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767378.795908:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767378.795912:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861473408:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767378.795931:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767378.795961:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767378.795969:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767378/real 1775767378] req@ffffa0e9d09dfb80 x1862023861473536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767378.795977:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861473536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767378.795983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767378.795985:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767378.795987:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473536:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767378.795991:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767378.796001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767378.796006:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767378/real 1775767378] req@ffffa0e9d09df100 x1862023861473664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767378.796013:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861473664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767378.796018:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767378.796020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767378.796022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473664:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767379.371662:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767379.627590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767380.395655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767380.651587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767381.419702:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767381.675651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767382.443594:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767382.699596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767383.467718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767383.723567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767383.915650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767383.915662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767383.915667:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767383.915678:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3112 00000100:00080000:2.0:1775767383.915689:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767383.915696:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767383.915737:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861473792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767383.915782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767383.915784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767383.915786:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767383.915789:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3112 00000100:00080000:2.0:1775767383.915791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767383.915794:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767383.915803:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767383.915811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861473920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767383.915818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767383.915820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767383.915821:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767383.915823:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3112 00000100:00080000:2.0:1775767383.915825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767383.915846:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767383.915851:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767383.915860:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861474048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767383.915888:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861473792:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767383.915982:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767383.916034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767383/real 1775767383] req@ffffa0e9d0712300 x1862023861473792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767383.916080:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861473792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767383.916092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767383.916097:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767383.916102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861473792:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767383.916110:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767383.916136:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767383.916148:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767383/real 1775767383] req@ffffa0e9d0713100 x1862023861473920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767383.916160:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861473920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767383.916168:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767383.916172:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767383.916175:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861473920:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767383.916202:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767383.916223:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767383.916235:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767383/real 1775767383] req@ffffa0e9d0711880 x1862023861474048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767383.916248:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861474048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767383.916279:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767383.916283:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767383.916287:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474048:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767384.491660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767384.747576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767385.515708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767385.771559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767386.539655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767386.795600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767387.563737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767387.819603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767388.587640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767388.843624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767389.035593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767389.035605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767389.035610:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767389.035621:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3117 00000100:00080000:2.0:1775767389.035633:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767389.035640:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767389.035683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861474176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767389.035712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767389.035715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767389.035716:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767389.035719:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3117 00000100:00080000:2.0:1775767389.035722:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767389.035745:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767389.035754:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767389.035765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861474304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767389.035771:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861474176:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767389.035775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767389.035781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767389.035782:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767389.035785:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3117 00000100:00080000:2.0:1775767389.035788:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767389.035790:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767389.035794:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767389.035802:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861474432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767389.035841:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767389.035895:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767389/real 1775767389] req@ffffa0e9d0711500 x1862023861474176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767389.035915:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861474176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767389.035927:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767389.035931:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767389.035935:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861474176:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767389.035965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767389.035989:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767389.036000:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767389/real 1775767389] req@ffffa0e9d0713480 x1862023861474304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767389.036008:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861474304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767389.036014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767389.036016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767389.036024:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474304:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767389.036028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767389.036042:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767389.036049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767389/real 1775767389] req@ffffa0e9d0712a00 x1862023861474432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767389.036057:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861474432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767389.036062:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767389.036064:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767389.036066:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474432:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767389.611637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767389.867650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767390.635655:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767390.891609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767391.659700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767391.915591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767392.683716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767392.939633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767393.707688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767393.963641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767394.155627:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767394.155637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767394.155642:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767394.155651:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3122 00000100:00080000:3.0:1775767394.155660:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767394.155666:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767394.155706:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861474560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767394.155731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767394.155733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767394.155735:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767394.155738:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3122 00000100:00080000:3.0:1775767394.155740:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767394.155743:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767394.155751:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767394.155761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861474688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767394.155769:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767394.155772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767394.155774:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767394.155779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861474560:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767394.155779:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3122 00000100:00080000:3.0:1775767394.155784:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767394.155789:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767394.155814:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767394.155825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861474816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767394.155850:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767394.155898:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767394/real 1775767394] req@ffffa0e9d0543800 x1862023861474560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767394.155913:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861474560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767394.155921:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767394.155924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767394.155928:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861474560:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767394.155950:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767394.155976:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767394.155986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767394/real 1775767394] req@ffffa0e9d0540700 x1862023861474688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767394.155994:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861474688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767394.156000:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767394.156003:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767394.156005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474688:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767394.156009:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767394.156019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767394.156025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767394/real 1775767394] req@ffffa0e9d0540e00 x1862023861474816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767394.156033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861474816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767394.156038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767394.156040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767394.156042:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861474816:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767394.731684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767394.987623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767395.755651:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767396.011612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767396.779663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767397.035566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767397.803706:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767398.059576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767398.827676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767399.083644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767399.275635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767399.275645:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767399.275650:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767399.275661:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3127 00000100:00080000:3.0:1775767399.275673:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767399.275681:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767399.275724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861474944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767399.275756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767399.275759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767399.275760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767399.275764:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3127 00000100:00080000:3.0:1775767399.275766:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767399.275768:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767399.275777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767399.275787:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861475072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767399.275795:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767399.275799:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767399.275800:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767399.275802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3127 00000100:00100000:1.0:1775767399.275820:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861474944:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767399.275821:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767399.275829:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767399.275835:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767399.275844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861475200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767399.275908:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767399.275957:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767399/real 1775767399] req@ffffa0e9d0540a80 x1862023861474944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767399.275983:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861474944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767399.275993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767399.275997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767399.276000:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861474944:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767399.276006:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767399.276024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767399.276032:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767399/real 1775767399] req@ffffa0e9d0542d80 x1862023861475072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767399.276041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861475072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767399.276046:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767399.276049:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767399.276051:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475072:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767399.276073:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767399.276083:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767399.276089:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767399/real 1775767399] req@ffffa0e9d0541f80 x1862023861475200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767399.276096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861475200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767399.276113:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767399.276115:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767399.276117:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475200:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767399.851655:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767400.107615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767400.875627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767401.131555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767401.899629:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767402.155621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767402.923684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767403.179587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767403.947620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767404.203582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767404.395598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767404.395609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767404.395613:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767404.395622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3132 00000100:00080000:3.0:1775767404.395632:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767404.395638:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767404.395678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861475328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767404.395704:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767404.395707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767404.395709:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767404.395711:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3132 00000100:00080000:3.0:1775767404.395714:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767404.395716:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767404.395725:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767404.395734:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861475456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767404.395742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767404.395751:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861475328:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767404.395766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767404.395768:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767404.395772:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3132 00000100:00080000:3.0:1775767404.395775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767404.395779:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767404.395784:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767404.395793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861475584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767404.395814:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767404.395852:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767404/real 1775767404] req@ffffa0e9d0541180 x1862023861475328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767404.395866:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861475328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767404.395874:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767404.395877:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767404.395880:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861475328:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767404.395901:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767404.395915:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767404.395931:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767404/real 1775767404] req@ffffa0e9d0543480 x1862023861475456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767404.395939:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861475456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767404.395944:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767404.395947:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767404.395956:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475456:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767404.395960:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767404.395973:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767404.395979:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767404/real 1775767404] req@ffffa0e9d0543100 x1862023861475584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767404.395986:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861475584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767404.395991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767404.395993:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767404.395995:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475584:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767404.971695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767405.227563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767405.995623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767406.251615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767407.019687:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767407.275729:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767408.043659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767408.299601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767409.067647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767409.323610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767409.515576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767409.515585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767409.515590:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767409.515599:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3138 00000100:00080000:3.0:1775767409.515611:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767409.515618:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767409.515661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861475712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767409.515705:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767409.515709:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767409.515711:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767409.515715:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3138 00000100:00080000:3.0:1775767409.515719:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767409.515724:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767409.515735:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767409.515749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861475840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767409.515770:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861475712:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767409.515780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767409.515783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767409.515785:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767409.515789:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3138 00000100:00080000:3.0:1775767409.515793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767409.515797:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767409.515802:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767409.515812:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861475968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767409.515844:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767409.515890:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767409/real 1775767409] req@ffffa0e9d0543100 x1862023861475712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767409.515906:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861475712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767409.515914:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767409.515917:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767409.515920:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861475712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767409.515942:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767409.515966:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767409.515975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767409/real 1775767409] req@ffffa0e9d0543480 x1862023861475840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767409.515983:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861475840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767409.516005:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767409.516007:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767409.516009:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475840:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767409.516013:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767409.516024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767409.516030:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767409/real 1775767409] req@ffffa0e9d0541180 x1862023861475968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767409.516039:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861475968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767409.516047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767409.516050:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767409.516054:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861475968:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767410.091712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767410.347604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767411.115708:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767411.371608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767412.139657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767412.395610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767413.163699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767413.419660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767414.187668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767414.443565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767414.635666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767414.635674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767414.635679:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767414.635689:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3143 00000100:00080000:1.0:1775767414.635719:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767414.635725:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767414.635763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861476096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767414.635794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767414.635797:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767414.635798:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767414.635801:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3143 00000100:00080000:1.0:1775767414.635803:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767414.635806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767414.635815:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767414.635823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861476224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767414.635831:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767414.635834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767414.635836:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767414.635839:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3143 00000100:00080000:1.0:1775767414.635841:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767414.635843:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767414.635846:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767414.635852:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861476352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767414.635859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861476096:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767414.635939:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767414.636002:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767414/real 1775767414] req@ffffa0e9c8deca80 x1862023861476096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767414.636023:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861476096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767414.636035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767414.636071:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767414.636076:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861476096:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767414.636092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476224:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767414.636120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767414.636135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767414/real 1775767414] req@ffffa0e9c8dedf80 x1862023861476224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767414.636149:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861476224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767414.636158:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767414.636161:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767414.636165:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767414.636193:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767414.636213:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767414.636223:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767414/real 1775767414] req@ffffa0e9c8dece00 x1862023861476352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767414.636235:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861476352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767414.636243:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767414.636247:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767414.636250:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476352:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767415.211701:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767415.467619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767416.235710:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767416.491582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767417.259722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767417.515649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767418.283632:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767418.539590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767419.307826:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767419.563607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767419.755635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767419.755651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767419.755656:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767419.755667:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3148 00000100:00080000:3.0:1775767419.755679:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767419.755687:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767419.755730:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861476480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767419.755761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767419.755764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767419.755765:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767419.755769:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3148 00000100:00080000:3.0:1775767419.755772:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767419.755775:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767419.755800:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767419.755809:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861476480:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767419.755814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861476608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767419.755824:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767419.755826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767419.755828:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767419.755831:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3148 00000100:00080000:3.0:1775767419.755833:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767419.755836:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767419.755839:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767419.755849:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861476736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767419.755879:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767419.755927:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767419/real 1775767419] req@ffffa0e9d0541f80 x1862023861476480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767419.755957:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861476480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767419.755969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767419.755974:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767419.755978:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861476480:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767419.756015:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767419.756036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767419.756050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767419/real 1775767419] req@ffffa0e9d0542d80 x1862023861476608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767419.756063:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861476608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767419.756072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767419.756076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767419.756080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476608:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767419.756086:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767419.756107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767419.756116:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767419/real 1775767419] req@ffffa0e9d0540a80 x1862023861476736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767419.756124:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861476736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767419.756139:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767419.756141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767419.756143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476736:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767420.331713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767420.587582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767421.355675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767421.611629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767422.379704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767422.635582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767423.403643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767423.659633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767424.427707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767424.683573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767424.875667:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767424.875675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767424.875679:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767424.875689:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3153 00000100:00080000:2.0:1775767424.875698:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767424.875705:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767424.875747:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861476864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767424.875777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767424.875781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767424.875783:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767424.875788:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3153 00000100:00080000:2.0:1775767424.875792:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767424.875817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767424.875824:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861476864:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775767424.875828:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767424.875844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861476992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767424.875853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767424.875856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767424.875858:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767424.875861:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3153 00000100:00080000:2.0:1775767424.875866:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767424.875870:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767424.875876:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767424.875889:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861477120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767424.875935:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476992:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767424.875979:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477120:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767425.451634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767425.707613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767426.475705:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767426.731628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767427.499661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767427.755612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775767427.947819:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775767427.947872:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767427.947892:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767427.948082:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767424/real 1775767427] req@ffffa0e9d0710e00 x1862023861476864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767449 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.948112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861476864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767449 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.948146:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.948152:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767427.948156:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861476864:192.168.202.104@tcp:250:kworker.0 00000800:00000100:3.0:1775767427.949437:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767427.949477:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767427.949532:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767424/real 1775767427] req@ffffa0e9d0710a80 x1862023861476992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.949566:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861476992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.949579:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.949585:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767427.949590:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861476992:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767427.949620:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767424/real 1775767427] req@ffffa0e9d0712680 x1862023861477120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.949636:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861477120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.949642:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.949645:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767427.949647:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477120:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767428.523704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767428.779631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767429.547693:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767429.803582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767429.995599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767429.995608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767429.995614:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767429.995640:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3158 00000100:00080000:2.0:1775767429.995654:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767429.995663:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767429.995711:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861477248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767429.995737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767429.995740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767429.995742:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767429.995745:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3158 00000100:00080000:2.0:1775767429.995748:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767429.995751:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767429.995760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767429.995774:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861477376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767429.995783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767429.995788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767429.995789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861477248:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767429.995789:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767429.995795:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3158 00000100:00080000:2.0:1775767429.995800:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767429.995803:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767429.995806:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767429.995816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861477504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767429.995885:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767429.995986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767429/real 1775767429] req@ffffa0e9d0712d80 x1862023861477248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767429.996023:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861477248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767429.996036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767429.996042:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767429.996046:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861477248:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767429.996092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767429.996120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767429.996135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767429/real 1775767429] req@ffffa0e9d0710700 x1862023861477376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767429.996149:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861477376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767429.996167:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767429.996171:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767429.996175:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477376:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767429.996181:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767429.996207:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767429.996219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767429/real 1775767429] req@ffffa0e9d0711f80 x1862023861477504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767429.996231:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861477504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767429.996240:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767429.996244:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767429.996247:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477504:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767430.571708:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767430.827655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767431.595669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767431.851571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767432.619652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767432.875732:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767433.643651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767433.899603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767434.667672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767434.923615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767435.115612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767435.115621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767435.115625:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767435.115635:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3163 00000100:00080000:2.0:1775767435.115646:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767435.115653:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767435.115692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861477632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767435.115717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767435.115720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767435.115721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767435.115724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3163 00000100:00080000:2.0:1775767435.115727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767435.115729:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767435.115738:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767435.115746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861477760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767435.115771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767435.115776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767435.115777:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861477632:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767435.115777:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767435.115783:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3163 00000100:00080000:2.0:1775767435.115788:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767435.115791:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767435.115795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767435.115804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861477888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767435.115873:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767435.115910:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767435/real 1775767435] req@ffffa0e9d0711c00 x1862023861477632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767435.115924:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861477632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767435.115932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767435.115935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767435.115943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861477632:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767435.115965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767435.115982:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767435.115990:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767435/real 1775767435] req@ffffa0e9d0713b80 x1862023861477760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767435.115998:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861477760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767435.116003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767435.116005:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767435.116007:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477760:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767435.116011:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767435.116020:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767435.116026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767435/real 1775767435] req@ffffa0e9d0710000 x1862023861477888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767435.116033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861477888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767435.116037:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767435.116039:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767435.116041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861477888:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767435.691611:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767435.947597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767436.715656:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767436.971621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767437.739638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767437.995601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767438.763665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767439.019601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767439.787614:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767440.043544:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767440.235642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767440.235653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767440.235677:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767440.235686:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3168 00000100:00080000:0.0:1775767440.235695:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767440.235701:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767440.235731:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861478016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767440.235762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767440.235767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767440.235768:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767440.235773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3168 00000100:00080000:0.0:1775767440.235777:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767440.235782:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767440.235793:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767440.235822:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861478144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767440.235834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767440.235838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767440.235840:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767440.235845:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3168 00000100:00080000:0.0:1775767440.235849:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767440.235852:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861478016:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767440.235856:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767440.235863:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767440.235876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861478272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767440.235914:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767440.235960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767440/real 1775767440] req@ffffa0e9d09dce00 x1862023861478016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767440.235978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861478016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767440.235990:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767440.235994:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767440.235998:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861478016:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767440.236006:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767440.236030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767440.236042:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767440/real 1775767440] req@ffffa0e9d09dc380 x1862023861478144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767440.236054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861478144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767440.236062:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767440.236064:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767440.236067:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478144:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767440.236086:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767440.236100:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767440.236115:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767440/real 1775767440] req@ffffa0e9d09dc000 x1862023861478272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767440.236123:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861478272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767440.236128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767440.236130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767440.236132:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478272:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767440.811654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767441.067592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767441.835704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767442.091597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767442.859654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767443.115590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767443.883669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767444.139542:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767444.907616:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767445.163600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767445.355599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767445.355608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767445.355614:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767445.355623:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3173 00000100:00080000:1.0:1775767445.355635:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767445.355641:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767445.355684:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861478400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767445.355714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767445.355717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767445.355718:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767445.355721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3173 00000100:00080000:1.0:1775767445.355723:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767445.355726:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767445.355735:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767445.355773:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861478528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767445.355782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767445.355786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767445.355789:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767445.355795:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3173 00000100:00100000:0.0:1775767445.355796:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861478400:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767445.355800:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767445.355809:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767445.355819:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767445.355828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861478656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767445.355892:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767445.355941:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767445/real 1775767445] req@ffffa0e9c8dec700 x1862023861478400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767445.355973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861478400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767445.355986:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767445.355991:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767445.355996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861478400:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767445.356024:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767445.356050:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767445.356063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767445/real 1775767445] req@ffffa0e9c8defb80 x1862023861478528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767445.356076:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861478528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767445.356085:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767445.356089:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767445.356093:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478528:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767445.356100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767445.356131:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767445.356159:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767445/real 1775767445] req@ffffa0e9c8def100 x1862023861478656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767445.356173:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861478656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767445.356181:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767445.356185:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767445.356188:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478656:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767445.931654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767446.187629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767446.955780:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767447.211590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767447.979692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767448.235611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767449.003641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767449.259564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767450.027724:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767450.283639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767450.475598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767450.475606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767450.475611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767450.475622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3178 00000100:00080000:1.0:1775767450.475633:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767450.475639:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767450.475680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861478784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767450.475730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767450.475734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767450.475736:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767450.475739:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3178 00000100:00080000:1.0:1775767450.475754:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767450.475759:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767450.475769:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767450.475781:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861478912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767450.475791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767450.475794:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861478784:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767450.475794:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767450.475798:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767450.475805:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3178 00000100:00080000:1.0:1775767450.475807:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767450.475810:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767450.475812:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767450.475819:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861479040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767450.475904:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767450.475961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767450/real 1775767450] req@ffffa0e9c8def100 x1862023861478784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.475984:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861478784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.475993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.475997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767450.476000:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861478784:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767450.476025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767450.476049:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767450.476063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767450/real 1775767450] req@ffffa0e9c8defb80 x1862023861478912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.476077:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861478912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.476100:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.476104:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767450.476107:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861478912:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767450.476113:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767450.476136:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767450.476147:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767450/real 1775767450] req@ffffa0e9c8dec700 x1862023861479040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.476160:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861479040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.476169:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.476173:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767450.476176:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479040:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767451.051634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767451.307568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767452.075628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767452.331601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767453.099657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767453.355605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767454.123744:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767454.379609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767455.147673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767455.403597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767455.595590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767455.595598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767455.595602:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767455.595611:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3184 00000100:00080000:2.0:1775767455.595620:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767455.595626:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767455.595676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861479168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767455.595698:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767455.595701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767455.595702:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767455.595705:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3184 00000100:00080000:2.0:1775767455.595708:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767455.595729:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767455.595739:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767455.595741:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861479168:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767455.595750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861479296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767455.595762:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767455.595766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767455.595768:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767455.595772:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3184 00000100:00080000:2.0:1775767455.595776:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767455.595780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767455.595786:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767455.595799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861479424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767455.595808:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767455.595852:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767455/real 1775767455] req@ffffa0e9d0710380 x1862023861479168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767455.595871:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861479168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767455.595882:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767455.595887:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767455.595899:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861479168:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767455.595923:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767455.595946:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767455.595957:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767455/real 1775767455] req@ffffa0e9d0711180 x1862023861479296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767455.595970:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861479296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767455.595979:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767455.595982:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767455.595986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479296:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767455.595991:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767455.596006:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767455.596014:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767455/real 1775767455] req@ffffa0e9d0710000 x1862023861479424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767455.596026:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861479424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767455.596034:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767455.596036:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767455.596039:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479424:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767456.171695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767456.427622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767457.195691:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767457.451581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767458.219675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767458.475589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767459.243708:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767459.499651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767460.267663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767460.523626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767460.715592:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767460.715623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767460.715628:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767460.715638:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3189 00000100:00080000:2.0:1775767460.715651:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767460.715680:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767460.715724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861479552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767460.715753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767460.715756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767460.715757:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767460.715760:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3189 00000100:00080000:2.0:1775767460.715764:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767460.715766:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767460.715776:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767460.715785:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861479680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767460.715793:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767460.715796:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767460.715797:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767460.715799:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3189 00000100:00080000:2.0:1775767460.715801:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767460.715804:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767460.715807:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767460.715814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861479808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767460.715826:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861479552:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767460.715907:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767460.715973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767460/real 1775767460] req@ffffa0e9d0713b80 x1862023861479552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767460.715997:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861479552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767460.716009:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767460.716014:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767460.716019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861479552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767460.716028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767460.716063:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767460.716078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767460/real 1775767460] req@ffffa0e9d0711c00 x1862023861479680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767460.716090:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861479680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767460.716098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767460.716101:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767460.716105:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479680:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767460.716134:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767460.716154:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767460.716166:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767460/real 1775767460] req@ffffa0e9d0711f80 x1862023861479808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767460.716180:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861479808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767460.716200:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767460.716204:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767460.716208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861479808:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767461.291690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767461.547606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767462.315674:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767462.571662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767463.339685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767463.595647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767464.363691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767464.619644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767465.387757:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767465.643622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767465.835644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767465.835653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767465.835658:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767465.835668:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3194 00000100:00080000:2.0:1775767465.835680:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767465.835687:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767465.835757:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861479936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767465.835784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767465.835787:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767465.835788:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767465.835791:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3194 00000100:00080000:2.0:1775767465.835793:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767465.835796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767465.835805:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767465.835813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861480064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767465.835820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767465.835823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767465.835839:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767465.835843:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3194 00000100:00080000:2.0:1775767465.835846:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767465.835849:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767465.835853:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767465.835862:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861479936:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767465.835865:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861480192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767465.835937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767465.835976:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767465/real 1775767465] req@ffffa0e9d0710700 x1862023861479936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767465.835991:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861479936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767465.836000:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767465.836004:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767465.836007:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861479936:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767465.836036:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767465.836054:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767465.836063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767465/real 1775767465] req@ffffa0e9d0712d80 x1862023861480064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767465.836071:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861480064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767465.836077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767465.836079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767465.836081:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480064:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767465.836112:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767465.836126:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767465.836133:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767465/real 1775767465] req@ffffa0e9d0712680 x1862023861480192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767465.836141:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861480192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767465.836146:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767465.836149:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767465.836151:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480192:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767466.411636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767466.667605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767467.435710:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767467.691665:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767468.459687:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767468.715583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767469.483735:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767469.739635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767470.507688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767470.763611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767470.955590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767470.955600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767470.955606:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767470.955618:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3199 00000100:00080000:2.0:1775767470.955632:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767470.955639:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767470.955684:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861480320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767470.955717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767470.955721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767470.955723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767470.955741:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3199 00000100:00080000:2.0:1775767470.955746:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767470.955752:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767470.955764:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767470.955780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861480448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767470.955793:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861480320:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767470.955798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767470.955804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767470.955806:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767470.955812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3199 00000100:00080000:2.0:1775767470.955817:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767470.955822:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767470.955828:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767470.955843:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861480576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767470.955866:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767470.955908:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767470/real 1775767470] req@ffffa0e9d0710a80 x1862023861480320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767470.955932:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861480320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767470.955941:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767470.955946:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767470.955950:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861480320:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767470.955976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767470.956001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767470.956020:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767470/real 1775767470] req@ffffa0e9d0710e00 x1862023861480448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767470.956042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861480448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767470.956048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767470.956051:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767470.956053:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480448:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767470.956057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767470.956073:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767470.956080:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767470/real 1775767470] req@ffffa0e9d0712a00 x1862023861480576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767470.956087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861480576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767470.956092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767470.956095:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767470.956097:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480576:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767471.531664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767471.787626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767472.555653:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767472.811595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767473.579697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767473.835641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767474.603628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767474.859617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767475.627665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767475.883627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767476.075577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767476.075586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767476.075609:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767476.075619:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3204 00000100:00080000:2.0:1775767476.075631:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767476.075637:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767476.075677:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861480704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767476.075723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767476.075726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767476.075728:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767476.075731:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3204 00000100:00080000:2.0:1775767476.075734:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767476.075736:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767476.075746:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767476.075755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861480832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767476.075763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767476.075767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767476.075769:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767476.075772:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3204 00000100:00080000:2.0:1775767476.075775:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767476.075778:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767476.075782:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767476.075790:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861480960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767476.075828:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861480704:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767476.075910:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767476.075977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767476/real 1775767476] req@ffffa0e9d0713480 x1862023861480704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767476.075999:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861480704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767476.076030:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767476.076035:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767476.076039:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861480704:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767476.076048:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767476.076075:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767476.076089:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767476/real 1775767476] req@ffffa0e9d0711500 x1862023861480832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767476.076103:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861480832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767476.076112:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767476.076116:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767476.076120:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480832:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767476.076149:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767476.076172:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767476.076185:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767476/real 1775767476] req@ffffa0e9d0711880 x1862023861480960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767476.076199:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861480960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767476.076209:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767476.076213:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767476.076217:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861480960:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767476.651623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767476.907590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767477.675623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767477.931609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767478.699688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767478.955586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767479.723692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767479.979588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767480.747677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767481.003562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767481.195614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767481.195624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767481.195628:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767481.195637:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3209 00000100:00080000:2.0:1775767481.195647:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767481.195654:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767481.195696:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861481088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767481.195728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767481.195731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767481.195733:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767481.195736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3209 00000100:00080000:2.0:1775767481.195739:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767481.195742:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767481.195751:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767481.195762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861481216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767481.195779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861481088:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767481.195794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767481.195797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767481.195798:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767481.195802:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3209 00000100:00080000:2.0:1775767481.195805:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767481.195808:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767481.195812:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767481.195824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861481344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767481.195844:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767481.195893:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767481/real 1775767481] req@ffffa0e9d0713100 x1862023861481088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767481.195913:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861481088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767481.195922:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767481.195926:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767481.195929:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861481088:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767481.195949:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767481.195967:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767481.195975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767481/real 1775767481] req@ffffa0e9d0712300 x1862023861481216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767481.195983:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861481216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767481.195989:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767481.195991:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767481.195993:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481216:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767481.195996:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767481.196011:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767481.196031:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767481/real 1775767481] req@ffffa0e9d0711880 x1862023861481344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767481.196052:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861481344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767481.196060:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767481.196064:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767481.196067:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481344:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767481.771659:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767482.027616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767482.795634:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767483.051554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767483.819663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767484.075602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767484.843689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767485.099613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767485.867689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767486.123598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767486.315604:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767486.315621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767486.315627:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767486.315638:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3214 00000100:00080000:1.0:1775767486.315648:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767486.315657:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767486.315694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861481472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767486.315717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767486.315720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767486.315721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767486.315724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3214 00000100:00080000:1.0:1775767486.315727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767486.315729:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767486.315738:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767486.315747:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861481600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767486.315754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767486.315759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767486.315760:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861481472:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767486.315760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767486.315767:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3214 00000100:00080000:1.0:1775767486.315770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767486.315772:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767486.315775:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767486.315800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861481728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767486.315838:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767486.315878:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767486/real 1775767486] req@ffffa0e9c8dece00 x1862023861481472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767486.315893:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861481472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767486.315902:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767486.315905:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767486.315908:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861481472:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767486.315935:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767486.315951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767486.315960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767486/real 1775767486] req@ffffa0e9c8dedf80 x1862023861481600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767486.315968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861481600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767486.315991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767486.315993:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767486.315996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481600:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767486.316000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767486.316014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767486.316021:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767486/real 1775767486] req@ffffa0e9c8deca80 x1862023861481728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767486.316030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861481728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767486.316038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767486.316041:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767486.316044:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481728:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767486.891678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767487.147645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767487.915701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767488.171610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767488.939623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767489.195602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767489.963659:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767490.219590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767490.987673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767491.243638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767491.435642:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767491.435655:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767491.435693:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767491.435702:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3219 00000100:00080000:2.0:1775767491.435712:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767491.435719:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767491.435761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861481856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767491.435794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767491.435797:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767491.435799:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767491.435802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3219 00000100:00080000:2.0:1775767491.435805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767491.435808:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767491.435817:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767491.435829:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861481984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767491.435837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767491.435840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767491.435842:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861481856:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767491.435842:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767491.435849:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3219 00000100:00080000:2.0:1775767491.435852:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767491.435855:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767491.435858:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767491.435868:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861482112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767491.435959:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481984:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767491.436004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482112:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767492.011699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767492.267605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767493.035684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767493.291619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767494.059650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767494.315604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775767494.507712:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775767494.507777:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767494.507799:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767494.507850:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767494.507861:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767494.507914:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767491/real 1775767494] req@ffffa0e9d0713480 x1862023861481984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767494.507947:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861481984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767494.507992:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767494.507999:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767494.508004:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861481984:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767494.508013:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767491/real 1775767494] req@ffffa0e9d0712a00 x1862023861482112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767494.508028:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861482112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767494.508038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767494.508043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767494.508047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482112:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767494.508055:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767491/real 1775767494] req@ffffa0e9d0711500 x1862023861481856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767494.508069:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861481856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767494.508077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767494.508081:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767494.508084:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861481856:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775767495.083646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767495.339635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767496.107623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767496.363565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767496.555665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767496.555674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767496.555677:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767496.555687:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3225 00000100:00080000:2.0:1775767496.555699:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767496.555706:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767496.555742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861482240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767496.555768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767496.555773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767496.555775:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767496.555779:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3225 00000100:00080000:2.0:1775767496.555799:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767496.555806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767496.555819:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767496.555840:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861482368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767496.555853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767496.555858:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767496.555861:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767496.555869:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3225 00000100:00100000:0.0:1775767496.555871:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861482240:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767496.555875:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767496.555886:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767496.555895:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767496.555913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861482496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767496.555949:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767496.556020:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767496/real 1775767496] req@ffffa0e9d0710e00 x1862023861482240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767496.556054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861482240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767496.556067:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767496.556073:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767496.556078:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861482240:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767496.556117:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767496.556145:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767496.556159:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767496/real 1775767496] req@ffffa0e9d0710a80 x1862023861482368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767496.556173:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861482368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767496.556181:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767496.556185:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767496.556188:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482368:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767496.556195:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767496.556223:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767496.556235:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767496/real 1775767496] req@ffffa0e9d0712680 x1862023861482496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767496.556247:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861482496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767496.556281:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767496.556285:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767496.556289:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482496:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767497.131712:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767497.387610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767498.155697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767498.411598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767499.179654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767499.435647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767500.203763:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767500.459542:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767501.227670:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767501.483614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767501.675677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767501.675686:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767501.675691:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767501.675702:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3230 00000100:00080000:0.0:1775767501.675715:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767501.675721:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767501.675776:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861482624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767501.675806:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767501.675809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767501.675811:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767501.675824:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3230 00000100:00080000:0.0:1775767501.675826:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767501.675829:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767501.675838:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767501.675848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861482752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767501.675856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767501.675859:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775767501.675870:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861482624:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767501.675878:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767501.675883:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3230 00000100:00080000:0.0:1775767501.675894:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767501.675897:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767501.675902:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767501.675913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861482880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767501.675954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767501.676003:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767501/real 1775767501] req@ffffa0e9d09de300 x1862023861482624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767501.676021:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861482624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767501.676030:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767501.676033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767501.676037:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861482624:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767501.676042:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767501.676058:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767501.676066:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767501/real 1775767501] req@ffffa0e9d09dea00 x1862023861482752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767501.676074:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861482752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767501.676079:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767501.676088:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767501.676091:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482752:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767501.676113:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767501.676143:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767501.676155:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767501/real 1775767501] req@ffffa0e9d09df480 x1862023861482880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767501.676181:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861482880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767501.676189:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767501.676193:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767501.676195:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861482880:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767502.251734:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767502.507622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767503.275671:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767503.531648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767504.299786:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767504.555620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767505.323662:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767505.579595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767506.347690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767506.603615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767506.795617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767506.795626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767506.795631:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767506.795643:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3235 00000100:00080000:3.0:1775767506.795672:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767506.795678:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767506.795719:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861483008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767506.795774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767506.795777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767506.795779:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767506.795783:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3235 00000100:00080000:3.0:1775767506.795787:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767506.795790:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767506.795802:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767506.795817:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861483136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767506.795819:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861483008:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767506.795837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767506.795840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767506.795841:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767506.795845:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3235 00000100:00080000:3.0:1775767506.795849:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767506.795852:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767506.795856:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767506.795866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861483264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767506.795906:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767506.795949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767506/real 1775767506] req@ffffa0e9d0540e00 x1862023861483008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767506.795983:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861483008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767506.795992:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767506.795995:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767506.795998:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861483008:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767506.796027:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767506.796065:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767506.796082:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767506/real 1775767506] req@ffffa0e9d0540700 x1862023861483136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767506.796093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861483136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767506.796100:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767506.796103:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767506.796105:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483136:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767506.796109:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767506.796123:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767506.796130:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767506/real 1775767506] req@ffffa0e9d0543800 x1862023861483264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767506.796139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861483264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767506.796145:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767506.796147:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767506.796149:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483264:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767507.371637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767507.627566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767508.395721:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767508.651550:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767509.419667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767509.675618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767510.443608:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767510.699595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767511.467639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767511.723591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767511.915564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767511.915584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767511.915589:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767511.915600:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3240 00000100:00080000:3.0:1775767511.915629:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767511.915636:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767511.915681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861483392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767511.915705:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767511.915708:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767511.915709:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767511.915713:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3240 00000100:00080000:3.0:1775767511.915715:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767511.915718:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767511.915728:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767511.915736:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861483520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767511.915743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767511.915746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767511.915748:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767511.915751:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3240 00000100:00080000:3.0:1775767511.915753:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767511.915756:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767511.915759:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767511.915766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861483648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767511.915773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861483392:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767511.915855:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767511.915917:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767511/real 1775767511] req@ffffa0e9d0542300 x1862023861483392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767511.915954:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861483392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767511.915963:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767511.915966:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767511.915970:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861483392:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767511.915976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767511.915997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767511.916006:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767511/real 1775767511] req@ffffa0e9d0540380 x1862023861483520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767511.916015:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861483520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767511.916031:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767511.916033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767511.916035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483520:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767511.916058:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767511.916068:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767511.916075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767511/real 1775767511] req@ffffa0e9d0541c00 x1862023861483648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767511.916093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861483648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767511.916126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767511.916129:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767511.916131:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483648:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767512.491692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767512.747590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767513.515645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767513.771561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767514.539670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767514.795596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767515.563743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767515.819574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767516.587643:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767516.843601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767517.035581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767517.035607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767517.035612:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767517.035622:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3245 00000100:00080000:3.0:1775767517.035634:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767517.035641:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767517.035678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861483776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767517.035712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767517.035714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767517.035716:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767517.035719:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3245 00000100:00080000:3.0:1775767517.035721:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767517.035724:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767517.035733:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767517.035742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861483904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767517.035750:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767517.035753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767517.035755:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767517.035758:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3245 00000100:00080000:3.0:1775767517.035761:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767517.035764:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767517.035767:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767517.035773:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861484032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767517.035808:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861483776:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767517.035892:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767517.035941:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767517/real 1775767517] req@ffffa0e9d0540000 x1862023861483776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767517.035957:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861483776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767517.035967:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767517.035972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767517.035975:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861483776:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767517.035981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767517.035996:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767517.036024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767517/real 1775767517] req@ffffa0e9d0542a00 x1862023861483904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767517.036032:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861483904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767517.036038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767517.036067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767517.036071:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861483904:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767517.036080:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767517.036133:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767517.036144:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767517/real 1775767517] req@ffffa0e9d0543b80 x1862023861484032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767517.036163:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861484032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767517.036169:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767517.036171:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767517.036173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484032:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767517.611789:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767517.867583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767518.635678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767518.891622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767519.659721:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767519.915569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767520.683636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767520.939579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767521.707729:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767521.963620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767522.155616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767522.155626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767522.155631:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767522.155640:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3250 00000100:00080000:0.0:1775767522.155649:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767522.155656:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767522.155699:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861484160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767522.155743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767522.155747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767522.155749:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767522.155754:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3250 00000100:00080000:0.0:1775767522.155758:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767522.155763:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767522.155775:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767522.155789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861484160:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767522.155792:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861484288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767522.155826:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767522.155828:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767522.155830:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767522.155833:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3250 00000100:00080000:0.0:1775767522.155845:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767522.155848:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767522.155853:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767522.155857:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767522.155867:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861484416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767522.155901:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767522/real 1775767522] req@ffffa0e9d09df480 x1862023861484160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767522.155915:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861484160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767522.155922:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767522.155926:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767522.155929:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861484160:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767522.155956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767522.155978:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767522.155992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767522/real 1775767522] req@ffffa0e9d09dea00 x1862023861484288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767522.156019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861484288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767522.156029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767522.156033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767522.156036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484288:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767522.156043:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767522.156060:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767522.156068:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767522/real 1775767522] req@ffffa0e9d09de300 x1862023861484416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767522.156076:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861484416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767522.156081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767522.156083:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767522.156085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484416:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767522.731705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767522.987610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767523.755658:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767524.011617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767524.779646:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767525.035562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767525.803626:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767526.059612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767526.827653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767527.083572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767527.275626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767527.275639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767527.275644:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767527.275655:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3255 00000100:00080000:2.0:1775767527.275666:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767527.275674:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767527.275717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861484544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767527.275750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767527.275755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767527.275757:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767527.275761:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3255 00000100:00080000:2.0:1775767527.275766:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767527.275770:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767527.275781:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767527.275796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861484672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767527.275805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861484544:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767527.275813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767527.275817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767527.275838:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767527.275845:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3255 00000100:00080000:2.0:1775767527.275851:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767527.275855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767527.275862:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767527.275874:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775767527.275878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861484800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767527.275945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767527/real 1775767527] req@ffffa0e9d0712d80 x1862023861484544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767527.275974:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861484544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767527.275983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767527.275988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767527.275990:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861484544:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767527.276009:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767527.276028:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767527.276036:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767527/real 1775767527] req@ffffa0e9d0710700 x1862023861484672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767527.276046:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861484672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767527.276055:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767527.276058:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767527.276061:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484672:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767527.276067:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767527.276087:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767527.276098:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767527/real 1775767527] req@ffffa0e9d0711f80 x1862023861484800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767527.276111:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861484800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767527.276138:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767527.276141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767527.276143:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861484800:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767527.851663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767528.107574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767528.875696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767529.131567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767529.899700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767530.155621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767530.923676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767531.179622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767531.947675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767532.203607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767532.395622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767532.395632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767532.395638:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767532.395650:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3260 00000100:00080000:1.0:1775767532.395663:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767532.395670:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767532.395716:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861484928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767532.395751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767532.395755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767532.395757:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767532.395761:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3260 00000100:00080000:1.0:1775767532.395764:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767532.395769:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767532.395780:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767532.395793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861485056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767532.395799:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861484928:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767532.395811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767532.395831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767532.395833:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767532.395839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3260 00000100:00080000:1.0:1775767532.395843:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767532.395848:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767532.395854:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767532.395866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861485184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767532.395889:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767532.395961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767532/real 1775767532] req@ffffa0e9c8ded880 x1862023861484928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767532.395982:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861484928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767532.395993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767532.395997:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767532.396001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861484928:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767532.396037:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767532.396061:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767532.396075:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767532/real 1775767532] req@ffffa0e9c8dee680 x1862023861485056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767532.396096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861485056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767532.396105:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767532.396108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767532.396111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485056:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767532.396118:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767532.396138:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767532.396147:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767532/real 1775767532] req@ffffa0e9c8deea00 x1862023861485184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767532.396159:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861485184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767532.396167:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767532.396170:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767532.396173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485184:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767532.971632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767533.227593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767533.995661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767534.251596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767535.019650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767535.275555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767536.043659:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767536.299585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767537.067668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767537.323656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767537.515669:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767537.515678:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767537.515683:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767537.515692:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3266 00000100:00080000:3.0:1775767537.515701:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767537.515707:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767537.515745:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861485312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767537.515770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767537.515773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767537.515775:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767537.515777:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3266 00000100:00080000:3.0:1775767537.515780:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767537.515782:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767537.515791:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767537.515799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861485440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767537.515807:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767537.515809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767537.515811:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767537.515813:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3266 00000100:00080000:3.0:1775767537.515815:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767537.515819:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861485312:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767537.515820:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767537.515843:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767537.515854:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861485568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767537.515900:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767537.515958:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767537/real 1775767537] req@ffffa0e9d0542680 x1862023861485312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767537.515979:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861485312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767537.516005:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767537.516009:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767537.516013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861485312:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767537.516021:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767537.516047:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767537.516062:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767537/real 1775767537] req@ffffa0e9d0541500 x1862023861485440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767537.516075:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861485440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767537.516084:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767537.516088:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767537.516091:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485440:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767537.516113:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767537.516129:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767537.516137:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767537/real 1775767537] req@ffffa0e9d0541880 x1862023861485568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767537.516144:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861485568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767537.516149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767537.516151:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767537.516153:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485568:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767538.091626:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767538.347571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767539.115663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767539.371577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767540.139630:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767540.395587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767541.163699:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767541.419612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767542.187703:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767542.443577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767542.635672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767542.635683:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767542.635689:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767542.635700:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3271 00000100:00080000:0.0:1775767542.635712:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767542.635720:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767542.635778:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861485696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767542.635848:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767542.635852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767542.635853:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767542.635856:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3271 00000100:00080000:0.0:1775767542.635859:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767542.635862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767542.635871:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767542.635882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861485824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767542.635890:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861485696:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767542.635892:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767542.635899:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767542.635900:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767542.635904:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3271 00000100:00080000:0.0:1775767542.635906:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767542.635908:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767542.635929:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767542.635938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861485952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767542.635958:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767542.635994:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767542/real 1775767542] req@ffffa0e9d09dc000 x1862023861485696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767542.636008:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861485696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767542.636016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767542.636020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767542.636023:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861485696:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767542.636041:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767542.636056:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767542.636063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767542/real 1775767542] req@ffffa0e9d09dc380 x1862023861485824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767542.636079:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861485824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767542.636097:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767542.636099:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767542.636101:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485824:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767542.636105:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767542.636117:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767542.636123:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767542/real 1775767542] req@ffffa0e9d09dce00 x1862023861485952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767542.636131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861485952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767542.636136:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767542.636138:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767542.636139:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861485952:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767543.211663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767543.467616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767544.235698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767544.491601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767545.259661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767545.515618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767546.283701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767546.539585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767547.307693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767547.563641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767547.755678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767547.755686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767547.755691:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767547.755701:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3276 00000100:00080000:1.0:1775767547.755713:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767547.755719:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767547.755798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861486080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767547.755830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767547.755833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767547.755834:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767547.755837:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3276 00000100:00080000:1.0:1775767547.755841:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767547.755844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767547.755854:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767547.755864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861486208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767547.755869:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861486080:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767547.755885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767547.755890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767547.755891:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767547.755897:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3276 00000100:00080000:1.0:1775767547.755902:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767547.755906:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767547.755913:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767547.755927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861486336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767547.755948:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767547.755989:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767547/real 1775767547] req@ffffa0e9c8dedc00 x1862023861486080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767547.756005:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861486080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767547.756013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767547.756016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767547.756019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861486080:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767547.756040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767547.756055:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767547.756063:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767547/real 1775767547] req@ffffa0e9c8def480 x1862023861486208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767547.756071:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861486208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767547.756077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767547.756079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767547.756081:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486208:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767547.756084:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767547.756107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767547.756114:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767547/real 1775767547] req@ffffa0e9c8def800 x1862023861486336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767547.756122:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861486336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767547.756127:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767547.756129:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767547.756130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486336:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767548.331679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767548.587597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767549.355707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767549.611604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767550.379689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767550.635591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767551.403684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767551.659548:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767552.427698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767552.683574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767552.875639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767552.875652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767552.875658:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767552.875669:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3281 00000100:00080000:2.0:1775767552.875679:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767552.875686:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767552.875761:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861486464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767552.875787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767552.875789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767552.875808:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767552.875811:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3281 00000100:00080000:2.0:1775767552.875814:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767552.875818:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767552.875828:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767552.875838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861486592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767552.875847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767552.875850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767552.875851:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767552.875854:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3281 00000100:00080000:2.0:1775767552.875856:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767552.875864:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767552.875866:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861486464:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775767552.875868:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767552.875884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861486720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767552.875955:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767552.876008:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767552/real 1775767552] req@ffffa0e9d0711c00 x1862023861486464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767552.876033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861486464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767552.876047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767552.876053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767552.876058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861486464:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767552.876087:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767552.876107:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767552.876116:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767552/real 1775767552] req@ffffa0e9d0713b80 x1862023861486592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767552.876144:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861486592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767552.876150:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767552.876162:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767552.876164:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486592:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767552.876169:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767552.876185:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767552.876192:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767552/real 1775767552] req@ffffa0e9d0710000 x1862023861486720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767552.876200:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861486720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767552.876205:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767552.876207:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767552.876209:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486720:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767553.451669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767553.707603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767554.475742:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767554.731611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767555.499654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767555.755598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767556.523689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767556.779598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767557.547703:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767557.803596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767557.995620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767557.995645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767557.995650:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767557.995673:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3286 00000100:00080000:1.0:1775767557.995687:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767557.995693:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767557.995734:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861486848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767557.995759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767557.995762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767557.995763:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767557.995766:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3286 00000100:00080000:1.0:1775767557.995769:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767557.995772:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767557.995782:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767557.995791:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861486976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767557.995796:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861486848:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767557.995802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767557.995806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767557.995807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767557.995811:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3286 00000100:00080000:1.0:1775767557.995812:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767557.995814:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767557.995818:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767557.995826:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861487104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767557.995911:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486976:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767557.995934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487104:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767558.571715:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767558.827544:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767559.595697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767559.851616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767560.619731:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767560.875672:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:1.0:1775767561.067892:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:1.0:1775767561.067968:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767561.067987:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775767561.068028:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775767561.068039:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767561.068157:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767557/real 1775767561] req@ffffa0e9c8ded500 x1862023861486976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767561.068248:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861486976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767561.068263:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767561.068269:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767561.068275:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861486976:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767561.068285:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767557/real 1775767561] req@ffffa0e9c8deed80 x1862023861487104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767561.068299:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861487104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767561.068308:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767561.068313:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767561.068316:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487104:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767561.068324:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767557/real 1775767561] req@ffffa0e9c8dec380 x1862023861486848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767561.068337:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861486848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767561.068344:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767561.068348:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767561.068351:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861486848:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775767561.643683:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767561.899566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767562.667660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767562.923630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767563.115692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767563.115702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767563.115707:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767563.115718:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3291 00000100:00080000:3.0:1775767563.115731:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767563.115739:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767563.115786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861487232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767563.115815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767563.115819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767563.115821:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767563.115826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3291 00000100:00080000:3.0:1775767563.115832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767563.115837:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767563.115853:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767563.115872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861487360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767563.115878:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861487232:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767563.115888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767563.115893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767563.115894:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767563.115899:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3291 00000100:00080000:3.0:1775767563.115921:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767563.115927:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767563.115934:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767563.115946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861487488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767563.116002:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767563.116071:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767563/real 1775767563] req@ffffa0e9d0541880 x1862023861487232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767563.116097:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861487232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767563.116109:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767563.116114:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767563.116119:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861487232:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767563.116149:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767563.116190:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767563.116206:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767563/real 1775767563] req@ffffa0e9d0541500 x1862023861487360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767563.116220:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861487360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767563.116230:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767563.116233:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767563.116237:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487360:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767563.116244:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767563.116264:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767563.116276:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767563/real 1775767563] req@ffffa0e9d0542680 x1862023861487488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767563.116304:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861487488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767563.116311:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767563.116314:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767563.116317:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487488:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767563.691715:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767563.947581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767564.715720:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767564.971647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767565.739734:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767565.995579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767566.763742:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767567.019652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767567.787761:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767568.043596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767568.235595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767568.235605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767568.235609:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767568.235620:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3296 00000100:00080000:3.0:1775767568.235631:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767568.235639:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767568.235680:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861487616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767568.235702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767568.235705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767568.235706:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767568.235709:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3296 00000100:00080000:3.0:1775767568.235712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767568.235715:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767568.235725:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767568.235734:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861487616:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767568.235735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861487744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767568.235748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767568.235765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767568.235766:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767568.235770:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3296 00000100:00080000:3.0:1775767568.235773:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767568.235776:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767568.235780:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767568.235789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861487872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767568.235811:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767568.235865:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767568/real 1775767568] req@ffffa0e9d0543b80 x1862023861487616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767568.235893:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861487616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767568.235903:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767568.235907:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767568.235910:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861487616:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767568.235932:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767568.235948:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767568.235957:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767568/real 1775767568] req@ffffa0e9d0542a00 x1862023861487744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767568.235965:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861487744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767568.235981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767568.235984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767568.235986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487744:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767568.235990:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767568.236004:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767568.236022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767568/real 1775767568] req@ffffa0e9d0540000 x1862023861487872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767568.236030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861487872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767568.236035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767568.236036:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767568.236038:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861487872:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767568.811705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767569.067648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767569.835687:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767570.091581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767570.859719:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767571.115635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767571.883662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767572.139606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767572.907669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767573.163621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767573.355618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767573.355627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767573.355631:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767573.355640:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3301 00000100:00080000:0.0:1775767573.355650:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767573.355655:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767573.355690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861488000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767573.355721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767573.355745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767573.355748:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767573.355753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3301 00000100:00080000:0.0:1775767573.355759:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767573.355763:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861488000:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767573.355767:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767573.355781:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767573.355794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861488128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767573.355803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767573.355806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767573.355807:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767573.355809:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3301 00000100:00080000:0.0:1775767573.355816:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767573.355818:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767573.355821:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767573.355825:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767573.355830:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861488256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767573.355864:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767573/real 1775767573] req@ffffa0e9d09df100 x1862023861488000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767573.355879:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861488000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767573.355887:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767573.355890:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767573.355893:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861488000:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767573.355911:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767573.355926:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767573.355934:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767573/real 1775767573] req@ffffa0e9d09dfb80 x1862023861488128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767573.355954:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861488128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767573.355960:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767573.355962:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767573.355964:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767573.355968:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767573.355981:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767573.355987:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767573/real 1775767573] req@ffffa0e9d09ded80 x1862023861488256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767573.355995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861488256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767573.355999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767573.356001:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767573.356003:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488256:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767573.931692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767574.187581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767574.955713:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767575.211600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767575.979668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767576.235574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767577.003662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767577.259624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767578.027691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767578.283547:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767578.475687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767578.475696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767578.475701:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767578.475712:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3306 00000100:00080000:0.0:1775767578.475729:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767578.475736:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767578.475778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861488384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767578.475816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767578.475819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767578.475821:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767578.475824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3306 00000100:00080000:0.0:1775767578.475827:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767578.475830:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767578.475840:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767578.475849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861488512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767578.475857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767578.475860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767578.475861:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767578.475866:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3306 00000100:00100000:1.0:1775767578.475866:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861488384:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767578.475869:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767578.475876:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767578.475879:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767578.475887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861488640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767578.475952:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767578.475992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767578/real 1775767578] req@ffffa0e9d09dc700 x1862023861488384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767578.476009:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861488384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767578.476029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767578.476034:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767578.476039:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861488384:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767578.476066:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767578.476090:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767578.476100:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767578/real 1775767578] req@ffffa0e9d09df800 x1862023861488512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767578.476109:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861488512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767578.476115:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767578.476118:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767578.476120:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488512:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767578.476123:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767578.476134:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767578.476140:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767578/real 1775767578] req@ffffa0e9d09dd880 x1862023861488640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767578.476147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861488640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767578.476152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767578.476154:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767578.476155:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488640:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767579.051674:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767579.307618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767580.075715:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767580.331612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767581.099635:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767581.355596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767582.123699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767582.379605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767583.147642:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767583.403574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767583.595595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767583.595605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767583.595609:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767583.595618:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3312 00000100:00080000:0.0:1775767583.595647:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767583.595653:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767583.595682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861488768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767583.595707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767583.595709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767583.595711:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767583.595714:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3312 00000100:00080000:0.0:1775767583.595716:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767583.595719:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767583.595726:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767583.595736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861488896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767583.595744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767583.595746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767583.595748:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767583.595750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3312 00000100:00080000:0.0:1775767583.595752:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767583.595754:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767583.595757:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767583.595765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861489024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767583.595773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861488768:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767583.595872:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767583.595922:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767583/real 1775767583] req@ffffa0e9d09dd500 x1862023861488768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767583.595945:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861488768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767583.595958:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767583.595962:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767583.595967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861488768:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767583.595976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767583.596010:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767583.596028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767583/real 1775767583] req@ffffa0e9d09dd180 x1862023861488896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767583.596040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861488896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767583.596049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767583.596052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767583.596056:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861488896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767583.596085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767583.596136:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767583.596150:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767583/real 1775767583] req@ffffa0e9d09ddc00 x1862023861489024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767583.596162:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861489024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767583.596170:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767583.596174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767583.596177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489024:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767584.171715:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767584.427635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767585.195693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767585.451588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767586.219710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767586.475623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767587.243665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767587.499629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767588.267680:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767588.523614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767588.715626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767588.715651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767588.715656:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767588.715666:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3317 00000100:00080000:0.0:1775767588.715678:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767588.715685:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767588.715734:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861489152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767588.715766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767588.715769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767588.715770:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767588.715773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3317 00000100:00080000:0.0:1775767588.715777:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767588.715780:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767588.715790:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767588.715802:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861489280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767588.715810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767588.715813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767588.715815:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767588.715820:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3317 00000100:00080000:0.0:1775767588.715824:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767588.715828:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767588.715834:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767588.715845:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861489152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767588.715854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861489408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767588.715934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767588.715991:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffffa0e9d09ddf80 x1862023861489152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767588.716013:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861489152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767588.716025:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767588.716031:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767588.716035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861489152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767588.716044:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767588.716069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767588.716083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffffa0e9d09dca80 x1862023861489280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767588.716117:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861489280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767588.716126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767588.716131:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767588.716134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489280:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767588.716162:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767588.716185:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767588.716197:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffffa0e9d09de680 x1862023861489408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767588.716211:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861489408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767588.716220:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767588.716223:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767588.716226:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489408:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767589.291615:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767589.547586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767590.315681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767590.571629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767591.339632:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767591.595577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767592.363670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767592.619635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767593.387686:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767593.643609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767593.835647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767593.835656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767593.835661:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767593.835671:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3322 00000100:00080000:3.0:1775767593.835695:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767593.835702:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767593.835746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861489536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767593.835778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767593.835783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767593.835785:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767593.835790:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3322 00000100:00080000:3.0:1775767593.835794:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767593.835800:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767593.835814:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767593.835824:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861489536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767593.835833:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861489664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767593.835856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767593.835858:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767593.835860:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767593.835864:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3322 00000100:00080000:3.0:1775767593.835868:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767593.835870:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767593.835875:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767593.835884:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861489792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767593.835911:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767593.835975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffffa0e9d0541c00 x1862023861489536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767593.835999:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861489536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767593.836011:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767593.836015:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767593.836031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861489536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767593.836057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767593.836081:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767593.836093:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffffa0e9d0540380 x1862023861489664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767593.836106:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861489664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767593.836115:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767593.836118:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767593.836122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767593.836128:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767593.836146:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767593.836164:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffffa0e9d0542300 x1862023861489792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767593.836176:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861489792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767593.836184:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767593.836187:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767593.836191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861489792:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767594.411686:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767594.667595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767595.435682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767595.691556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767596.459627:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767596.715590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767597.483667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767597.739571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767598.507646:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767598.763563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767598.955606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767598.955614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767598.955620:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767598.955629:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3327 00000100:00080000:1.0:1775767598.955668:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767598.955675:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767598.955720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861489920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767598.955755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767598.955760:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767598.955762:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767598.955767:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3327 00000100:00080000:1.0:1775767598.955771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767598.955777:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767598.955790:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767598.955803:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861490048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767598.955811:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861489920:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767598.955814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767598.955819:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767598.955821:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767598.955825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3327 00000100:00080000:1.0:1775767598.955828:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767598.955831:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767598.955835:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767598.955844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861490176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767598.955908:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767598.955954:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767598/real 1775767598] req@ffffa0e9c8dee300 x1862023861489920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767598.955974:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861489920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767598.955985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767598.955989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767598.955993:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861489920:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767598.956033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767598.956048:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767598.956057:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767598/real 1775767598] req@ffffa0e9c8dec000 x1862023861490048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767598.956065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861490048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767598.956081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767598.956084:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767598.956086:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767598.956090:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767598.956104:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767598.956110:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767598/real 1775767598] req@ffffa0e9c8ded180 x1862023861490176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767598.956118:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861490176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767598.956123:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767598.956125:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767598.956127:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490176:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767599.531665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767599.787662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767600.555655:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767600.811598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767601.579696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767601.835602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767602.603693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767602.859577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767603.627688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767603.883593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767604.075648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767604.075659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767604.075664:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767604.075674:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3332 00000100:00080000:2.0:1775767604.075685:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767604.075693:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767604.075735:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861490304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767604.075764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767604.075766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767604.075768:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767604.075771:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3332 00000100:00080000:2.0:1775767604.075774:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767604.075776:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767604.075785:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767604.075794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861490432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767604.075817:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861490304:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767604.075833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767604.075836:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767604.075838:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767604.075844:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3332 00000100:00080000:2.0:1775767604.075849:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767604.075853:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767604.075860:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767604.075875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861490560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767604.075878:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767604.075932:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767604/real 1775767604] req@ffffa0e9d0711180 x1862023861490304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767604.075948:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861490304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767604.075956:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767604.075960:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767604.075968:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861490304:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767604.075989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767604.076008:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767604.076017:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767604/real 1775767604] req@ffffa0e9d0710380 x1862023861490432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767604.076025:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861490432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767604.076030:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767604.076033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767604.076035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490432:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767604.076038:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767604.076048:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767604.076054:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767604/real 1775767604] req@ffffa0e9d0710000 x1862023861490560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767604.076061:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861490560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767604.076073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767604.076075:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767604.076077:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490560:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767604.651657:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767604.907573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767605.675632:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767605.931608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767606.699631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767606.955614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767607.723649:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767607.979616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767608.747646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767609.003629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767609.195592:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767609.195604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767609.195611:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767609.195624:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3337 00000100:00080000:1.0:1775767609.195639:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767609.195648:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767609.195699:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861490688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767609.195730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767609.195735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767609.195737:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767609.195741:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3337 00000100:00080000:1.0:1775767609.195746:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767609.195752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767609.195765:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767609.195776:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861490688:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767609.195782:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861490816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767609.195800:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767609.195804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767609.195806:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767609.195811:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3337 00000100:00080000:1.0:1775767609.195816:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767609.195820:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767609.195843:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767609.195858:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861490944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767609.195863:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767609.195910:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767609/real 1775767609] req@ffffa0e9c8ded180 x1862023861490688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767609.195926:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861490688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767609.195934:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767609.195937:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767609.195941:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861490688:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767609.195961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767609.195976:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767609.195984:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767609/real 1775767609] req@ffffa0e9c8dec000 x1862023861490816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767609.195992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861490816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767609.195998:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767609.196000:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767609.196003:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490816:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767609.196006:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767609.196015:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767609.196020:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767609/real 1775767609] req@ffffa0e9c8dee300 x1862023861490944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767609.196040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861490944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767609.196046:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767609.196048:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767609.196050:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861490944:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767609.771692:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767610.027626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767610.795624:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767611.051591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767611.819685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767612.075626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767612.843667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767613.099595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767613.867657:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767614.123610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767614.315584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767614.315596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767614.315601:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767614.315611:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3342 00000100:00080000:1.0:1775767614.315621:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767614.315627:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767614.315661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861491072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767614.315695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767614.315699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767614.315701:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767614.315706:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3342 00000100:00080000:1.0:1775767614.315710:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767614.315714:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767614.315725:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767614.315746:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861491072:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767614.315749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861491200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767614.315763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767614.315766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767614.315787:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767614.315792:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3342 00000100:00080000:1.0:1775767614.315797:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767614.315802:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775767614.315808:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775767614.315809:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767614.315821:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861491328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767614.315858:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767614/real 1775767614] req@ffffa0e9c8dec380 x1862023861491072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767614.315876:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861491072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767614.315884:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767614.315887:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767614.315890:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861491072:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767614.315909:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767614.315926:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767614.315934:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767614/real 1775767614] req@ffffa0e9c8deed80 x1862023861491200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767614.315942:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861491200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767614.315958:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767614.315961:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767614.315963:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767614.315967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767614.315981:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767614.315988:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767614/real 1775767614] req@ffffa0e9c8ded500 x1862023861491328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767614.316000:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861491328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767614.316008:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767614.316011:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767614.316014:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491328:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767614.891661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767615.147578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767615.915659:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767616.171618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767616.939667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767617.195593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767617.963614:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767618.219634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767618.987704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767619.243620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767619.435589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767619.435599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767619.435603:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767619.435612:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3347 00000100:00080000:1.0:1775767619.435622:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767619.435629:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767619.435672:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861491456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767619.435702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767619.435705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767619.435706:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767619.435710:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3347 00000100:00080000:1.0:1775767619.435713:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767619.435716:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767619.435724:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767619.435734:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861491584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767619.435759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767619.435762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767619.435764:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767619.435767:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3347 00000100:00080000:1.0:1775767619.435770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767619.435773:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767619.435777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767619.435783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861491712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767619.435805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861491456:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767619.435882:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767619.435935:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767619/real 1775767619] req@ffffa0e9c8def800 x1862023861491456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767619.435956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861491456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767619.435965:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767619.435983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767619.435987:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861491456:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767619.435994:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767619.436035:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767619.436045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767619/real 1775767619] req@ffffa0e9c8def480 x1862023861491584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767619.436054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861491584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767619.436060:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767619.436062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767619.436085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491584:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767619.436116:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767619.436142:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767619.436155:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767619/real 1775767619] req@ffffa0e9c8dedc00 x1862023861491712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767619.436166:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861491712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767619.436172:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767619.436174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767619.436177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491712:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767620.011603:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767620.267681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767621.035691:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767621.291681:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767622.059667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767622.315585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767623.083618:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767623.339646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767624.107646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767624.363601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767624.555569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767624.555578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767624.555583:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767624.555593:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3353 00000100:00080000:1.0:1775767624.555605:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767624.555611:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767624.555651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861491840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767624.555677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767624.555679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767624.555681:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767624.555684:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3353 00000100:00080000:1.0:1775767624.555701:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767624.555705:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767624.555714:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767624.555724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861491968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767624.555733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767624.555736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767624.555737:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767624.555740:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3353 00000100:00080000:1.0:1775767624.555742:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767624.555746:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767624.555751:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861491840:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775767624.555751:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767624.555759:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861492096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767624.555880:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491968:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767624.555931:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492096:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767625.131642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767625.387622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767626.155681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767626.411630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767627.179647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767627.435620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775767627.627781:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775767627.627822:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767627.627840:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767627.627887:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767627.627898:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767627.627969:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767624/real 1775767627] req@ffffa0e9c8ded880 x1862023861492096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767627.628041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861492096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767627.628054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767627.628060:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767627.628065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492096:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767627.628074:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767624/real 1775767627] req@ffffa0e9c8deea00 x1862023861491840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767627.628087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861491840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767627.628094:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767627.628097:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767627.628101:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861491840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767627.628107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767624/real 1775767627] req@ffffa0e9c8dee680 x1862023861491968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767627.628149:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861491968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767627.628158:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767627.628163:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767627.628170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861491968:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767628.203636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767628.459593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767629.227681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767629.483627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767629.675671:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767629.675680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767629.675684:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767629.675694:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3358 00000100:00080000:2.0:1775767629.675703:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767629.675710:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767629.675750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861492224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767629.675792:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767629.675816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767629.675819:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767629.675826:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3358 00000100:00100000:0.0:1775767629.675832:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861492224:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767629.675844:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767629.675852:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767629.675864:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767629.675883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861492352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767629.675898:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767629.675902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767629.675904:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767629.675909:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3358 00000100:00080000:2.0:1775767629.675914:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767629.675917:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:2.0:1775767629.675928:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767629.675937:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767629.675953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861492480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767629.676009:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffffa0e9d0713b80 x1862023861492224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767629.676038:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861492224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767629.676054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767629.676059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767629.676065:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861492224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767629.676098:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767629.676125:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767629.676141:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffffa0e9d0711c00 x1862023861492352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767629.676171:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861492352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767629.676202:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767629.676206:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767629.676210:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492352:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767629.676218:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767629.676244:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767629.676259:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffffa0e9d0711f80 x1862023861492480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767629.676301:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861492480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767629.676310:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767629.676313:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767629.676316:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492480:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767630.251677:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767630.507596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767631.275616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767631.531616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767632.299710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767632.555573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767633.323667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767633.579630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767634.347666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767634.603581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767634.795633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767634.795645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767634.795651:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767634.795661:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3363 00000100:00080000:0.0:1775767634.795672:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767634.795679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767634.795725:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861492608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767634.795782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767634.795786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767634.795789:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767634.795793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3363 00000100:00080000:0.0:1775767634.795798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767634.795814:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767634.795825:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767634.795837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861492736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767634.795845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767634.795846:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861492608:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767634.795849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767634.795852:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767634.795856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3363 00000100:00080000:0.0:1775767634.795858:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767634.795861:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767634.795864:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767634.795872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861492864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767634.795952:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767634.795991:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767634/real 1775767634] req@ffffa0e9d09de680 x1862023861492608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767634.796006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861492608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767634.796025:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767634.796028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767634.796032:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861492608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767634.796051:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767634.796069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767634.796077:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767634/real 1775767634] req@ffffa0e9d09dca80 x1862023861492736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767634.796085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861492736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767634.796090:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767634.796092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767634.796094:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492736:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767634.796098:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767634.796108:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767634.796113:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767634/real 1775767634] req@ffffa0e9d09ddf80 x1862023861492864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767634.796120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861492864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767634.796125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767634.796127:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767634.796128:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861492864:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767635.371677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767635.627617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767636.395669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767636.651644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767637.419657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767637.675617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767638.443636:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767638.699584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767639.467665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767639.723630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767639.915661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767639.915672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767639.915676:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767639.915708:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3368 00000100:00080000:0.0:1775767639.915717:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767639.915728:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767639.915761:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861492992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767639.915785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767639.915788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767639.915789:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767639.915792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3368 00000100:00080000:0.0:1775767639.915794:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767639.915796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767639.915805:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767639.915813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861493120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767639.915820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767639.915823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767639.915824:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767639.915826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3368 00000100:00080000:0.0:1775767639.915828:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767639.915830:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767639.915833:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767639.915842:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861493248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767639.915844:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861492992:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767639.915929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767639.915971:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767639/real 1775767639] req@ffffa0e9d09ddc00 x1862023861492992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767639.915991:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861492992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767639.916003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767639.916008:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767639.916012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861492992:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767639.916020:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767639.916040:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767639.916052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767639/real 1775767639] req@ffffa0e9d09dd180 x1862023861493120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767639.916066:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861493120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767639.916075:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767639.916079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767639.916083:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493120:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767639.916105:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767639.916133:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767639.916144:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767639/real 1775767639] req@ffffa0e9d09dd500 x1862023861493248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767639.916158:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861493248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767639.916166:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767639.916170:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767639.916173:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493248:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767640.491676:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767640.747613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767641.515678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767641.771591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767642.539674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767642.795603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767643.563715:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767643.819618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767644.587664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767644.843608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767645.035559:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767645.035568:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767645.035572:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767645.035580:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3373 00000100:00080000:0.0:1775767645.035589:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767645.035600:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767645.035644:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861493376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767645.035673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767645.035677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767645.035679:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767645.035683:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3373 00000100:00080000:0.0:1775767645.035688:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767645.035692:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767645.035703:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767645.035722:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861493504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767645.035734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767645.035738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767645.035740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767645.035744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3373 00000100:00080000:0.0:1775767645.035747:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767645.035757:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861493376:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767645.035771:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767645.035776:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767645.035788:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861493632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767645.035865:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767645.035910:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767645/real 1775767645] req@ffffa0e9d09dd880 x1862023861493376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767645.035926:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861493376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767645.035935:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767645.035939:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767645.035942:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861493376:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767645.035948:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767645.035964:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767645.035972:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767645/real 1775767645] req@ffffa0e9d09df800 x1862023861493504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767645.035993:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861493504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767645.035999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767645.036002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767645.036004:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493504:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767645.036026:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767645.036040:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767645.036046:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767645/real 1775767645] req@ffffa0e9d09dc700 x1862023861493632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767645.036054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861493632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767645.036059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767645.036062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767645.036064:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493632:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767645.611638:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767645.867593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767646.635666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767646.891624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767647.659647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767647.915612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767648.683683:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767648.939615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767649.707658:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767649.963576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767650.155643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767650.155652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767650.155658:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767650.155668:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3378 00000100:00080000:2.0:1775767650.155680:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767650.155687:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767650.155741:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861493760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767650.155791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767650.155795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767650.155812:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767650.155816:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3378 00000100:00080000:2.0:1775767650.155820:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767650.155824:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767650.155834:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767650.155848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861493888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767650.155849:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861493760:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767650.155864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767650.155866:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767650.155868:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767650.155871:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3378 00000100:00080000:2.0:1775767650.155874:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767650.155876:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767650.155879:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767650.155888:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861494016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767650.155936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767650.155981:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767650/real 1775767650] req@ffffa0e9d0710700 x1862023861493760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767650.156005:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861493760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767650.156019:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767650.156023:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767650.156028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861493760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767650.156056:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767650.156076:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767650.156085:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767650/real 1775767650] req@ffffa0e9d0712d80 x1862023861493888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767650.156094:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861493888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767650.156099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767650.156102:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767650.156104:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861493888:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767650.156108:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767650.156122:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767650.156131:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767650/real 1775767650] req@ffffa0e9d0712680 x1862023861494016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767650.156139:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861494016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767650.156145:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767650.156147:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767650.156149:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494016:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767650.731679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767650.987574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767651.755613:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767652.011607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767652.779691:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767653.035598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767653.803681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767654.059567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767654.827660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767655.083617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767655.275629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767655.275638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767655.275643:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767655.275654:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3383 00000100:00080000:1.0:1775767655.275664:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767655.275671:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767655.275713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861494144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767655.275764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767655.275767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767655.275769:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767655.275774:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3383 00000100:00080000:1.0:1775767655.275777:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767655.275782:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767655.275792:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767655.275806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861494272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767655.275819:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767655.275827:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861494144:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767655.275827:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767655.275829:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767655.275850:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3383 00000100:00080000:1.0:1775767655.275854:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767655.275877:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767655.275884:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767655.275912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861494400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767655.275920:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767655.275972:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767655/real 1775767655] req@ffffa0e9c8deca80 x1862023861494144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767655.275989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861494144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767655.275999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767655.276002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767655.276005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861494144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767655.276028:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767655.276053:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767655.276062:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767655/real 1775767655] req@ffffa0e9c8dedf80 x1862023861494272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767655.276070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861494272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767655.276091:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767655.276093:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767655.276096:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767655.276101:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767655.276117:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767655.276124:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767655/real 1775767655] req@ffffa0e9c8dece00 x1862023861494400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767655.276131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861494400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767655.276137:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767655.276139:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767655.276141:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494400:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767655.851737:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767656.107609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767656.875694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767657.131601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767657.899644:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767658.155610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767658.923649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767659.179584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767659.947701:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767660.203644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767660.395680:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767660.395690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767660.395694:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767660.395704:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3388 00000100:00080000:0.0:1775767660.395716:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767660.395723:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767660.395766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861494528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767660.395795:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767660.395816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767660.395819:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767660.395825:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3388 00000100:00080000:0.0:1775767660.395830:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767660.395831:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861494528:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767660.395838:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767660.395850:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767660.395866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861494656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767660.395877:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767660.395880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767660.395882:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767660.395885:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3388 00000100:00080000:0.0:1775767660.395896:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767660.395898:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767660.395902:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767660.395904:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767660.395915:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861494784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767660.395946:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffffa0e9d09ded80 x1862023861494528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767660.395961:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861494528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767660.395969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767660.395973:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767660.395976:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861494528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767660.395998:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767660.396012:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767660.396020:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffffa0e9d09dfb80 x1862023861494656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767660.396028:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861494656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767660.396034:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767660.396043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767660.396045:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494656:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767660.396049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767660.396063:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767660.396070:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffffa0e9d09df100 x1862023861494784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767660.396078:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861494784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767660.396082:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767660.396084:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767660.396086:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861494784:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767660.971779:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767661.227659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767661.995690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767662.251623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767663.019733:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767663.275600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767664.043679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767664.299625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767665.067725:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767665.323601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767665.515636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767665.515647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767665.515654:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767665.515665:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3394 00000100:00080000:2.0:1775767665.515679:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767665.515688:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767665.515730:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861494912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767665.515757:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767665.515759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767665.515761:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767665.515764:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3394 00000100:00080000:2.0:1775767665.515767:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767665.515769:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767665.515777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767665.515786:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861495040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767665.515814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767665.515818:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775767665.515819:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861494912:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767665.515820:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767665.515828:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3394 00000100:00080000:2.0:1775767665.515831:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767665.515834:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767665.515838:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767665.515848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861495168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767665.515894:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767665.515934:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffffa0e9d0710a80 x1862023861494912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767665.515952:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861494912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767665.515964:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767665.515968:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767665.515973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861494912:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767665.516000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767665.516031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767665.516042:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffffa0e9d0710e00 x1862023861495040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767665.516051:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861495040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767665.516057:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767665.516060:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767665.516062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767665.516066:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767665.516078:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767665.516083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffffa0e9d0711500 x1862023861495168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767665.516091:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861495168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767665.516096:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767665.516098:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767665.516099:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495168:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767666.091705:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767666.347629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767667.115667:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767667.371658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767668.139658:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767668.395666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767669.163635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767669.419615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767670.187743:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767670.443613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767670.635656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767670.635665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767670.635670:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767670.635679:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3399 00000100:00080000:2.0:1775767670.635691:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767670.635697:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767670.635742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861495296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767670.635770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767670.635788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767670.635789:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767670.635793:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3399 00000100:00080000:2.0:1775767670.635796:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767670.635801:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767670.635811:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767670.635824:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861495424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767670.635825:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861495296:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767670.635840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767670.635843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767670.635845:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767670.635848:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3399 00000100:00080000:2.0:1775767670.635850:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767670.635852:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767670.635856:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767670.635863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861495552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767670.635907:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767670.635956:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767670/real 1775767670] req@ffffa0e9d0712a00 x1862023861495296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767670.635979:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861495296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767670.635988:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767670.635991:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767670.635994:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861495296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767670.636016:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767670.636034:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767670.636044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767670/real 1775767670] req@ffffa0e9d0713480 x1862023861495424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767670.636053:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861495424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767670.636058:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767670.636060:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767670.636062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495424:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767670.636065:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767670.636077:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767670.636083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767670/real 1775767670] req@ffffa0e9d0711880 x1862023861495552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767670.636091:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861495552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767670.636105:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767670.636107:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767670.636110:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495552:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767671.211738:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767671.467600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767672.235692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767672.491620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767673.259706:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767673.515630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767674.283647:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767674.539666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767675.307731:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767675.563715:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767675.755635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767675.755644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767675.755667:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767675.755676:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3404 00000100:00080000:2.0:1775767675.755687:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767675.755693:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767675.755731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861495680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767675.755762:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767675.755765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767675.755766:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767675.755769:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3404 00000100:00080000:2.0:1775767675.755772:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767675.755774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767675.755783:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767675.755791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861495808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767675.755799:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767675.755802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767675.755803:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767675.755806:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3404 00000100:00080000:2.0:1775767675.755808:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767675.755810:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767675.755814:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767675.755820:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861495680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767675.755823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861495936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767675.755894:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767675.755933:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffffa0e9d0712300 x1862023861495680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767675.755949:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861495680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767675.755958:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767675.755962:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767675.755965:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861495680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767675.755984:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767675.756014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767675.756028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffffa0e9d0713100 x1862023861495808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767675.756042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861495808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767675.756051:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767675.756063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767675.756067:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495808:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767675.756097:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767675.756117:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767675.756125:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffffa0e9d0711880 x1862023861495936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767675.756144:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861495936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767675.756149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767675.756152:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767675.756154:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861495936:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767676.331654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767676.587607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767677.355707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767677.611646:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767678.379675:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767678.635641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767679.403683:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767679.659630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767680.427629:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767680.683588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767680.875567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767680.875576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767680.875580:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767680.875589:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3409 00000100:00080000:2.0:1775767680.875599:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767680.875604:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767680.875639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861496064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767680.875663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767680.875666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767680.875667:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767680.875670:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3409 00000100:00080000:2.0:1775767680.875673:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767680.875675:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767680.875683:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767680.875692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861496192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767680.875700:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767680.875714:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861496064:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767680.875724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767680.875726:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767680.875731:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3409 00000100:00080000:2.0:1775767680.875736:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767680.875741:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767680.875747:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767680.875759:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861496320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767680.875786:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767680.875830:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffffa0e9d0713480 x1862023861496064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767680.875850:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861496064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767680.875862:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767680.875866:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767680.875871:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861496064:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767680.875879:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767680.875909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767680.875923:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffffa0e9d0712a00 x1862023861496192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767680.875937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861496192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767680.875947:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767680.875951:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767680.875955:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767680.875979:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767680.876000:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767680.876012:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffffa0e9d0711500 x1862023861496320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767680.876026:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861496320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767680.876035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767680.876039:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767680.876043:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496320:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767681.451650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767681.707563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767682.475669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767682.731612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767683.499707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767683.755609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767684.523683:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767684.779614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767685.547680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767685.803602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767685.995627:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767685.995636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767685.995641:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767685.995651:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3414 00000100:00080000:0.0:1775767685.995663:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767685.995669:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767685.995712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861496448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767685.995740:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767685.995743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767685.995744:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767685.995747:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3414 00000100:00080000:0.0:1775767685.995750:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767685.995752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767685.995761:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767685.995769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861496576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767685.995777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767685.995780:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767685.995782:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767685.995784:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3414 00000100:00080000:0.0:1775767685.995786:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767685.995788:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767685.995801:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861496448:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767685.995808:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767685.995818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861496704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767685.995875:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767685.995920:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767685/real 1775767685] req@ffffa0e9d09dce00 x1862023861496448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767685.995944:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861496448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767685.995953:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767685.995956:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767685.995959:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861496448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767685.995965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767685.995983:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767685.995993:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767685/real 1775767685] req@ffffa0e9d09dc380 x1862023861496576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767685.996001:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861496576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767685.996007:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767685.996010:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767685.996012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496576:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767685.996033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767685.996044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767685.996051:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767685/real 1775767685] req@ffffa0e9d09dc000 x1862023861496704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767685.996058:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861496704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767685.996077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767685.996081:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767685.996085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496704:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767686.571694:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767686.827598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767687.595665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767687.851619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767688.619676:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767688.875571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767689.643689:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767689.899556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767690.667723:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767690.923572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767691.115626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767691.115635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767691.115640:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767691.115648:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3419 00000100:00080000:2.0:1775767691.115657:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767691.115663:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767691.115716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861496832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767691.115741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767691.115745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767691.115746:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767691.115750:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3419 00000100:00080000:2.0:1775767691.115753:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767691.115756:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767691.115764:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767691.115775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861496960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767691.115782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767691.115785:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767691.115786:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767691.115789:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3419 00000100:00080000:2.0:1775767691.115791:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767691.115795:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861496832:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767691.115796:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767691.115803:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767691.115811:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861497088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767691.115898:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496960:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767691.115935:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497088:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767691.691661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767691.947601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767692.715654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767692.971614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767693.739633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767693.995563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775767694.187787:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775767694.187836:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767694.187850:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767694.187895:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767694.187909:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775767694.188031:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767691/real 1775767694] req@ffffa0e9d0712680 x1862023861497088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767694.188058:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861497088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767694.188070:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767694.188076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767694.188080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767694.188088:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767691/real 1775767694] req@ffffa0e9d0710e00 x1862023861496832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767694.188100:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861496832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767694.188138:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767694.188143:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767694.188147:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861496832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767694.188155:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767691/real 1775767694] req@ffffa0e9d0710a80 x1862023861496960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767694.188172:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861496960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767694.188180:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767694.188184:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767694.188187:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861496960:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767694.763675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767695.019634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767695.787629:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767696.043586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767696.235589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767696.235600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767696.235604:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767696.235614:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3424 00000100:00080000:2.0:1775767696.235623:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767696.235629:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767696.235663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861497216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767696.235698:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767696.235701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767696.235703:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767696.235705:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3424 00000100:00080000:2.0:1775767696.235709:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767696.235713:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767696.235724:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767696.235739:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861497344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767696.235753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767696.235760:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861497216:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767696.235760:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767696.235762:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767696.235771:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3424 00000100:00080000:2.0:1775767696.235775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767696.235807:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767696.235814:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767696.235828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861497472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767696.235853:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767696.235904:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767696/real 1775767696] req@ffffa0e9d0712d80 x1862023861497216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767696.235924:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861497216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767696.235937:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767696.235942:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767696.235947:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861497216:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767696.235976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767696.236017:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767696.236032:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767696/real 1775767696] req@ffffa0e9d0710700 x1862023861497344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767696.236044:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861497344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767696.236053:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767696.236056:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767696.236060:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497344:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767696.236066:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767696.236087:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767696.236099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767696/real 1775767696] req@ffffa0e9d0711f80 x1862023861497472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767696.236129:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861497472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767696.236137:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767696.236141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767696.236144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497472:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767696.811637:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767697.067614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767697.835696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767698.091563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767698.859678:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767699.115630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767699.883700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767700.139573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767700.907680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767701.163593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767701.355669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767701.355681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767701.355686:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767701.355720:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3429 00000100:00080000:1.0:1775767701.355732:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767701.355740:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767701.355778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861497600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767701.355812:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767701.355816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767701.355818:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767701.355823:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3429 00000100:00080000:1.0:1775767701.355827:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767701.355840:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767701.355852:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767701.355856:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861497600:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767701.355869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861497728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767701.355882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767701.355886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767701.355888:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767701.355893:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3429 00000100:00080000:1.0:1775767701.355911:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767701.355915:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:1.0:1775767701.355918:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767701.355925:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767701.355940:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861497856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767701.355958:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767701] req@ffffa0e9c8dec700 x1862023861497600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767701.355984:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861497600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767701.355992:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767701.355996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767701.356000:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861497600:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767701.356019:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767701.356034:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767701.356042:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767701] req@ffffa0e9c8defb80 x1862023861497728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767701.356050:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861497728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767701.356066:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767701.356069:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767701.356071:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497728:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767701.356075:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767701.356088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767701.356094:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767701] req@ffffa0e9c8def100 x1862023861497856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767701.356101:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861497856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767701.356106:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767701.356108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767701.356109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861497856:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767701.931657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767702.187561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767702.955723:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767703.211636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767703.979727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767704.235606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767705.003637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767705.259655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767706.027630:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767706.283585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767706.475566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767706.475579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767706.475584:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767706.475595:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3434 00000100:00080000:1.0:1775767706.475608:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767706.475615:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767706.475660:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861497984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767706.475687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767706.475691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767706.475692:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767706.475696:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3434 00000100:00080000:1.0:1775767706.475700:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767706.475703:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767706.475714:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767706.475726:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861498112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767706.475737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767706.475741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767706.475743:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767706.475744:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861497984:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767706.475750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3434 00000100:00080000:1.0:1775767706.475773:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767706.475778:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767706.475784:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767706.475797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861498240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767706.475810:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767706.475855:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767706/real 1775767706] req@ffffa0e9c8def100 x1862023861497984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767706.475872:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861497984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767706.475880:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767706.475883:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767706.475893:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861497984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767706.475913:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767706.475929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767706.475938:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767706/real 1775767706] req@ffffa0e9c8defb80 x1862023861498112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767706.475946:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861498112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767706.475951:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767706.475953:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767706.475955:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498112:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767706.475959:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767706.475968:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767706.475973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767706/real 1775767706] req@ffffa0e9c8dec700 x1862023861498240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767706.475980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861498240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767706.475985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767706.475986:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767706.475988:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498240:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767707.051653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767707.307631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767708.075660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767708.331579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767709.099656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767709.355598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767710.123683:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767710.379585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767711.147610:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767711.403558:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767711.595597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767711.595608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767711.595612:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767711.595621:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3440 00000100:00080000:0.0:1775767711.595632:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767711.595640:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767711.595682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861498368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767711.595712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767711.595715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767711.595717:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767711.595720:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3440 00000100:00080000:0.0:1775767711.595723:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767711.595727:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767711.595736:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767711.595747:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861498368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767711.595748:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861498496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767711.595761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767711.595763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767711.595764:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767711.595767:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3440 00000100:00080000:0.0:1775767711.595769:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767711.595772:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767711.595775:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767711.595798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861498624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767711.595814:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767711.595851:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767711/real 1775767711] req@ffffa0e9d09de300 x1862023861498368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767711.595871:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861498368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767711.595883:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767711.595888:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767711.595892:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861498368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767711.595916:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767711.595937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767711.595947:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767711/real 1775767711] req@ffffa0e9d09dea00 x1862023861498496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767711.595955:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861498496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767711.595960:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767711.595970:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767711.595972:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498496:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767711.595976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767711.595990:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767711.595997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767711/real 1775767711] req@ffffa0e9d09df480 x1862023861498624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767711.596005:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861498624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767711.596009:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767711.596011:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767711.596013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498624:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767712.171722:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767712.427631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767713.195665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767713.451620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767714.219677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767714.475621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767715.243650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767715.499592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767716.267671:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767716.523604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767716.715566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767716.715578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767716.715583:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767716.715594:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3445 00000100:00080000:0.0:1775767716.715606:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767716.715614:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767716.715659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861498752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767716.715685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767716.715688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767716.715689:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767716.715692:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3445 00000100:00080000:0.0:1775767716.715695:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767716.715698:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767716.715707:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767716.715717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861498880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767716.715725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767716.715729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775767716.715730:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861498752:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767716.715731:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767716.715753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3445 00000100:00080000:0.0:1775767716.715756:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767716.715759:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767716.715780:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767716.715792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861499008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767716.715815:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767716.715857:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767716/real 1775767716] req@ffffa0e9d09df480 x1862023861498752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767716.715873:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861498752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767716.715883:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767716.715886:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767716.715890:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861498752:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767716.715918:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767716.715938:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767716.715949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767716/real 1775767716] req@ffffa0e9d09dea00 x1862023861498880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767716.715958:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861498880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767716.715965:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767716.715967:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767716.715970:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861498880:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767716.715974:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767716.715987:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767716.715993:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767716/real 1775767716] req@ffffa0e9d09de300 x1862023861499008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767716.716004:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861499008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767716.716014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767716.716018:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767716.716021:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499008:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767717.291664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767717.547596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767718.315672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767718.571628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767719.339680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767719.595582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767720.363675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767720.619588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767721.387625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767721.643596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767721.835654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767721.835666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767721.835671:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767721.835682:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3450 00000100:00080000:2.0:1775767721.835694:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767721.835713:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767721.835755:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861499136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767721.835786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767721.835799:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767721.835801:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767721.835806:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3450 00000100:00080000:2.0:1775767721.835811:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767721.835817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767721.835828:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767721.835841:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861499136:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767721.835846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861499264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767721.835872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767721.835876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767721.835885:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767721.835891:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3450 00000100:00080000:2.0:1775767721.835896:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767721.835901:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767721.835907:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767721.835921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861499392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767721.835934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767721.835984:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767721/real 1775767721] req@ffffa0e9d0711c00 x1862023861499136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767721.835999:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861499136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767721.836007:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767721.836010:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767721.836013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861499136:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767721.836034:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767721.836052:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767721.836069:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767721/real 1775767721] req@ffffa0e9d0713b80 x1862023861499264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767721.836092:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861499264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767721.836098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767721.836100:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767721.836102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767721.836106:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767721.836122:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767721.836129:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767721/real 1775767721] req@ffffa0e9d0710000 x1862023861499392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767721.836147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861499392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767721.836152:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767721.836155:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767721.836157:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499392:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767722.411668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767722.667612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767723.435669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767723.691610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767724.459725:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767724.715579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767725.483639:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767725.739596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767726.507637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767726.763626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767726.955576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767726.955587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767726.955592:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767726.955603:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3455 00000100:00080000:2.0:1775767726.955616:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767726.955623:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767726.955664:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861499520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767726.955691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767726.955695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767726.955697:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767726.955701:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3455 00000100:00080000:2.0:1775767726.955738:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767726.955743:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767726.955746:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861499520:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775767726.955754:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767726.955766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861499648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767726.955777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767726.955780:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767726.955782:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767726.955785:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3455 00000100:00080000:2.0:1775767726.955788:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767726.955791:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767726.955795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767726.955806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861499776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767726.955807:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767726.955852:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767726/real 1775767726] req@ffffa0e9d0710380 x1862023861499520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767726.955869:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861499520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767726.955877:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767726.955881:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767726.955885:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861499520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767726.955918:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767726.955944:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767726.955954:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767726/real 1775767726] req@ffffa0e9d0711180 x1862023861499648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767726.955963:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861499648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767726.955986:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767726.955988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767726.955991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499648:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767726.955997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767726.956019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767726.956045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767726/real 1775767726] req@ffffa0e9d0710000 x1862023861499776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767726.956059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861499776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767726.956067:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767726.956070:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767726.956073:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861499776:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767727.531715:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767727.787625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767728.555651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767728.811576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767729.579735:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767729.835629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767730.603680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767730.859626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767731.627694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767731.883578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767732.075643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767732.075656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767732.075660:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767732.075671:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3460 00000100:00080000:0.0:1775767732.075681:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767732.075687:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767732.075723:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861499904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767732.075749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767732.075752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767732.075754:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767732.075756:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3460 00000100:00080000:0.0:1775767732.075759:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767732.075761:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767732.075797:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861499904:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767732.075798:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767732.075823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861500032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767732.075836:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767732.075839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767732.075842:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767732.075846:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3460 00000100:00080000:0.0:1775767732.075851:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767732.075855:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775767732.075860:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767732.075867:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767732.075893:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861500160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767732.075905:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767732/real 1775767732] req@ffffa0e9d09dc000 x1862023861499904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767732.075921:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861499904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767732.075929:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767732.075939:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767732.075942:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861499904:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767732.075962:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767732.075999:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767732.076009:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767732/real 1775767732] req@ffffa0e9d09dc380 x1862023861500032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767732.076017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861500032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767732.076023:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767732.076025:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767732.076027:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767732.076031:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767732.076043:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767732.076049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767732/real 1775767732] req@ffffa0e9d09dce00 x1862023861500160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767732.076056:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861500160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767732.076061:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767732.076063:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767732.076064:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500160:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767732.651637:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767732.907562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767733.675709:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767733.931623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767734.699647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767734.955610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767735.723641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767735.979607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767736.747685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767737.003638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767737.195570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767737.195580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767737.195584:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767737.195593:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3465 00000100:00080000:0.0:1775767737.195602:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767737.195608:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767737.195644:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861500288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767737.195673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767737.195675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767737.195694:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767737.195697:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3465 00000100:00080000:0.0:1775767737.195701:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767737.195705:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767737.195714:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767737.195725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861500416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767737.195733:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861500288:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767737.195736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767737.195742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767737.195743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767737.195747:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3465 00000100:00080000:0.0:1775767737.195749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767737.195751:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767737.195754:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767737.195763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861500544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767737.195806:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767737.195859:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767737/real 1775767737] req@ffffa0e9d09df100 x1862023861500288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767737.195876:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861500288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767737.195884:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767737.195887:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767737.195890:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861500288:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767737.195909:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767737.196072:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767737.196087:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767737/real 1775767737] req@ffffa0e9d09dfb80 x1862023861500416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767737.196100:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861500416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767737.196109:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767737.196113:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767737.196117:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500416:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767737.196124:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767737.196143:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767737.196151:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767737/real 1775767737] req@ffffa0e9d09ded80 x1862023861500544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767737.196172:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861500544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767737.196180:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767737.196184:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767737.196187:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500544:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767737.771693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767738.027613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767738.795681:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767739.051550:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767739.819690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767740.075619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767740.843668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767741.099596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767741.867693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767742.123629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767742.315607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767742.315617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767742.315621:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767742.315629:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3470 00000100:00080000:0.0:1775767742.315639:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767742.315645:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767742.315698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861500672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767742.315723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767742.315725:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767742.315727:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767742.315729:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3470 00000100:00080000:0.0:1775767742.315732:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767742.315735:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767742.315743:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767742.315750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861500800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767742.315757:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767742.315759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767742.315760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767742.315763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3470 00000100:00080000:0.0:1775767742.315765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767742.315766:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767742.315769:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767742.315775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861500928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767742.315806:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861500672:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767742.315863:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767742.315922:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767742/real 1775767742] req@ffffa0e9d09dc700 x1862023861500672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767742.315937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861500672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767742.315945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767742.315948:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767742.315951:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861500672:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767742.315956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767742.315972:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767742.315980:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767742/real 1775767742] req@ffffa0e9d09df800 x1862023861500800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767742.315987:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861500800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767742.316005:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767742.316009:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767742.316013:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500800:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767742.316018:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767742.316042:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767742.316054:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767742/real 1775767742] req@ffffa0e9d09dd880 x1862023861500928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767742.316066:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861500928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767742.316073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767742.316076:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767742.316080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861500928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767742.891618:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767743.147588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767743.915674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767744.171567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767744.939631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767745.195624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767745.963684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767746.219621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767746.987598:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767747.243630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767747.435612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767747.435638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767747.435644:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767747.435653:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3475 00000100:00080000:0.0:1775767747.435666:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767747.435672:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767747.435712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861501056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767747.435749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767747.435753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767747.435755:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767747.435760:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3475 00000100:00080000:0.0:1775767747.435763:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767747.435767:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767747.435777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767747.435789:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861501184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767747.435798:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767747.435803:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767747.435804:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767747.435805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861501056:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767747.435809:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3475 00000100:00080000:0.0:1775767747.435817:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767747.435822:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767747.435837:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767747.435850:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861501312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767747.435908:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767747.435948:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffffa0e9d09dd500 x1862023861501056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767747.435965:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861501056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767747.435973:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767747.435983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767747.435986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861501056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767747.436009:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767747.436034:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767747.436043:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffffa0e9d09dd180 x1862023861501184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767747.436051:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861501184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767747.436057:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767747.436059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767747.436062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501184:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767747.436065:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767747.436076:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767747.436081:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffffa0e9d09ddc00 x1862023861501312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767747.436089:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861501312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767747.436094:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767747.436096:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767747.436097:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501312:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767748.011633:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767748.267622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767749.035684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767749.291589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767750.059673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767750.315602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767751.083699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767751.339569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767752.107651:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767752.363570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767752.555586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767752.555595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767752.555599:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767752.555608:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3481 00000100:00080000:1.0:1775767752.555617:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767752.555623:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767752.555659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861501440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767752.555691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767752.555695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767752.555697:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767752.555703:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3481 00000100:00080000:1.0:1775767752.555708:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767752.555711:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767752.555720:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767752.555727:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861501440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767752.555734:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861501568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767752.555742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767752.555745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767752.555746:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767752.555749:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3481 00000100:00080000:1.0:1775767752.555751:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767752.555753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767752.555794:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767752.555804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861501696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767752.555813:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767752.555860:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffffa0e9c8dece00 x1862023861501440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767752.555876:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861501440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767752.555884:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767752.555887:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767752.555890:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861501440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767752.555908:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767752.555924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767752.555932:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffffa0e9c8dedf80 x1862023861501568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767752.555939:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861501568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767752.555945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767752.555947:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767752.555949:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501568:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767752.555952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767752.555974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767752.555981:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffffa0e9c8deca80 x1862023861501696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767752.555989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861501696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767752.555994:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767752.555996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767752.555998:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501696:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767753.131689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767753.387613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767754.155631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767754.411561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767755.179706:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767755.435643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767756.203685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767756.459601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767757.227704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767757.483621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767757.675639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767757.675651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767757.675656:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767757.675667:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3486 00000100:00080000:1.0:1775767757.675679:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767757.675686:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767757.675745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861501824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767757.675776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767757.675780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767757.675782:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767757.675786:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3486 00000100:00080000:1.0:1775767757.675790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767757.675794:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767757.675805:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767757.675820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861501952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767757.675824:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861501824:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767757.675841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767757.675845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767757.675865:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767757.675870:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3486 00000100:00080000:1.0:1775767757.675876:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767757.675880:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767757.675886:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767757.675899:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861502080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767757.675936:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501952:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767757.675967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767758.251678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767758.507552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767759.275737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767759.531613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767760.299616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767760.555618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775767760.747788:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775767760.747834:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767760.747852:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767760.747902:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767760.747913:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767760.748007:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767760] req@ffffa0e9c8deea00 x1862023861501952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767760.748047:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861501952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767760.748061:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767760.748066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767760.748070:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861501952:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767760.748104:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767760] req@ffffa0e9c8ded880 x1862023861502080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767760.748120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861502080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767760.748127:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767760.748131:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767760.748134:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502080:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767760.748141:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767760] req@ffffa0e9c8dee680 x1862023861501824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767760.748154:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861501824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767760.748162:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767760.748166:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767760.748170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861501824:192.168.202.104@tcp:250:kworker.0 00010000:00010000:1.0:1775767761.323648:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767761.579609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767762.347622:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767762.603583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767762.795632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767762.795641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767762.795646:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767762.795656:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3491 00000100:00080000:0.0:1775767762.795667:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767762.795674:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767762.795725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861502208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767762.795777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767762.795781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767762.795783:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767762.795790:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3491 00000100:00080000:0.0:1775767762.795813:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767762.795822:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767762.795834:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861502208:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767762.795842:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767762.795863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861502336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767762.795877:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767762.795881:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767762.795883:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767762.795888:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3491 00000100:00080000:0.0:1775767762.795907:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767762.795918:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767762.795929:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767762.795932:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767762.795950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861502464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767762.795992:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffffa0e9d09ddf80 x1862023861502208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767762.796016:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861502208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767762.796029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767762.796034:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767762.796038:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861502208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767762.796069:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767762.796097:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767762.796119:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffffa0e9d09dca80 x1862023861502336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767762.796138:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861502336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767762.796168:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767762.796172:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767762.796175:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767762.796183:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767762.796207:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767762.796219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffffa0e9d09de680 x1862023861502464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767762.796232:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861502464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767762.796241:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767762.796244:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767762.796249:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502464:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767763.371701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767763.627605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767764.395635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767764.651575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767765.419743:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767765.675624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767766.443623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767766.699604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767767.467704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767767.723656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767767.915621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767767.915644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767767.915650:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767767.915663:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3496 00000100:00080000:2.0:1775767767.915677:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767767.915686:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767767.915723:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861502592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767767.915751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767767.915754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767767.915756:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767767.915758:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3496 00000100:00080000:2.0:1775767767.915761:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767767.915786:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767767.915795:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767767.915800:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861502592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767767.915809:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861502720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767767.915818:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767767.915820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767767.915822:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767767.915824:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3496 00000100:00080000:2.0:1775767767.915827:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767767.915830:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767767.915833:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767767.915841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861502848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767767.915861:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767767.915898:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767767] req@ffffa0e9d0713b80 x1862023861502592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767767.915911:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861502592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767767.915919:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767767.915929:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767767.915932:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861502592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767767.915952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767767.915969:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767767.915977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767767] req@ffffa0e9d0711c00 x1862023861502720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767767.915985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861502720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767767.915990:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767767.915992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767767.915994:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502720:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767767.915998:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767767.916007:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767767.916012:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767767] req@ffffa0e9d0711f80 x1862023861502848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767767.916019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861502848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767767.916024:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767767.916026:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767767.916028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861502848:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767768.491729:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767768.747599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767769.515682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767769.771634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767770.539694:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767770.795629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767771.563643:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767771.819585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767772.587700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767772.843648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767773.035666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767773.035676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767773.035697:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767773.035708:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3501 00000100:00080000:0.0:1775767773.035719:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767773.035732:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767773.035779:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861502976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767773.035812:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767773.035815:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767773.035817:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767773.035820:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3501 00000100:00080000:0.0:1775767773.035823:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767773.035826:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767773.035836:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767773.035850:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861503104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767773.035859:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767773.035863:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767773.035864:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767773.035866:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3501 00000100:00080000:0.0:1775767773.035871:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767773.035877:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767773.035881:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861502976:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767773.035882:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767773.035897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861503232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767773.035954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767773.035994:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767773/real 1775767773] req@ffffa0e9d09de680 x1862023861502976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767773.036020:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861502976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767773.036033:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767773.036037:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767773.036042:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861502976:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767773.036049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767773.036072:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767773.036084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767773/real 1775767773] req@ffffa0e9d09dca80 x1862023861503104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767773.036096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861503104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767773.036103:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767773.036106:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767773.036109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503104:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767773.036132:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767773.036153:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767773.036163:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767773/real 1775767773] req@ffffa0e9d09ddf80 x1862023861503232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767773.036193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861503232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767773.036201:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767773.036204:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767773.036206:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503232:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767773.611662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767773.867600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767774.635718:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767774.891627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767775.659653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767775.915594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767776.683695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767776.939644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767777.707655:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767777.963609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767778.155619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767778.155629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767778.155634:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767778.155642:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3506 00000100:00080000:0.0:1775767778.155651:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767778.155658:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767778.155692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861503360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767778.155713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767778.155715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767778.155716:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767778.155720:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3506 00000100:00080000:0.0:1775767778.155723:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767778.155726:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767778.155735:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767778.155747:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861503488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767778.155748:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861503360:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767778.155765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767778.155768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767778.155769:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767778.155772:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3506 00000100:00080000:0.0:1775767778.155805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767778.155808:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767778.155813:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767778.155815:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767778.155826:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861503616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767778.155854:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767778/real 1775767778] req@ffffa0e9d09ddc00 x1862023861503360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767778.155874:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861503360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767778.155882:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767778.155885:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767778.155888:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861503360:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767778.155905:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767778.155921:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767778.155929:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767778/real 1775767778] req@ffffa0e9d09dd180 x1862023861503488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767778.155937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861503488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767778.155949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767778.155952:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767778.155954:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503488:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767778.155961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767778.155982:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767778.155993:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767778/real 1775767778] req@ffffa0e9d09dd500 x1862023861503616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767778.156006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861503616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767778.156015:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767778.156018:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767778.156022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503616:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767778.731765:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767778.987613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767779.755696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767780.011619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767780.779707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767781.035604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767781.803703:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767782.059625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767782.827682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767783.083620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767783.275638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767783.275647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767783.275652:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767783.275661:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3511 00000100:00080000:2.0:1775767783.275672:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767783.275678:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767783.275718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861503744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767783.275744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767783.275747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767783.275749:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767783.275751:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3511 00000100:00080000:2.0:1775767783.275754:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767783.275756:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767783.275764:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767783.275772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861503872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767783.275780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767783.275783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775767783.275805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861503744:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767783.275805:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767783.275812:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3511 00000100:00080000:2.0:1775767783.275816:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767783.275819:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767783.275822:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767783.275831:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861504000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767783.275880:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767783.275923:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767783/real 1775767783] req@ffffa0e9d0710700 x1862023861503744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767783.275945:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861503744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767783.275957:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767783.275968:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767783.275973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861503744:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767783.275981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767783.276004:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767783.276017:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767783/real 1775767783] req@ffffa0e9d0712d80 x1862023861503872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767783.276030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861503872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767783.276050:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767783.276054:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767783.276058:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861503872:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767783.276084:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767783.276104:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767783.276116:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767783/real 1775767783] req@ffffa0e9d0710a80 x1862023861504000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767783.276128:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861504000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767783.276137:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767783.276141:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767783.276144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504000:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767783.851663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767784.107652:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767784.875689:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767785.131600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767785.899625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767786.155562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767786.923721:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767787.179590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767787.947698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767788.203570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767788.395643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767788.395653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767788.395657:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767788.395666:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3516 00000100:00080000:1.0:1775767788.395676:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767788.395682:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767788.395719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861504128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767788.395748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767788.395751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767788.395753:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767788.395755:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3516 00000100:00080000:1.0:1775767788.395758:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767788.395760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767788.395768:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767788.395780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861504256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767788.395790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767788.395794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767788.395796:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767788.395800:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3516 00000100:00080000:1.0:1775767788.395804:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767788.395808:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767788.395813:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861504128:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775767788.395815:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767788.395850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861504384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767788.395888:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767788.395946:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffffa0e9c8dedc00 x1862023861504128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767788.395968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861504128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767788.395979:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767788.395983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767788.395988:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861504128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767788.395995:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767788.396017:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767788.396028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffffa0e9c8def480 x1862023861504256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767788.396041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861504256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767788.396062:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767788.396066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767788.396069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767788.396093:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767788.396112:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767788.396123:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffffa0e9c8def800 x1862023861504384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767788.396135:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861504384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767788.396143:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767788.396146:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767788.396149:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504384:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767788.971650:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767789.227608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767789.995684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767790.251568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767791.019598:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767791.275598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767792.043710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767792.299630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767793.067638:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767793.323576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767793.515574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767793.515583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767793.515587:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767793.515595:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3522 00000100:00080000:1.0:1775767793.515605:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767793.515611:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767793.515647:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861504512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767793.515673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767793.515676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767793.515678:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767793.515681:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3522 00000100:00080000:1.0:1775767793.515683:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767793.515685:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767793.515693:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767793.515703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861504640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767793.515710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767793.515712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767793.515713:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767793.515735:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3522 00000100:00080000:1.0:1775767793.515738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767793.515741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767793.515745:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767793.515748:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861504512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767793.515755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861504768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767793.515823:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767793.515890:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767793/real 1775767793] req@ffffa0e9c8ded500 x1862023861504512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767793.515910:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861504512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767793.515919:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767793.515928:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767793.515931:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861504512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767793.515937:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767793.515964:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767793.515973:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767793/real 1775767793] req@ffffa0e9c8deed80 x1862023861504640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767793.515982:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861504640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767793.515987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767793.515989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767793.515992:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504640:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767793.516010:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767793.516022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767793.516028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767793/real 1775767793] req@ffffa0e9c8dec380 x1862023861504768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767793.516035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861504768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767793.516041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767793.516042:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767793.516044:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861504768:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767794.091670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767794.347634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767795.115652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767795.371584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767796.139666:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767796.395611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767797.163609:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767797.419581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767798.187643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767798.443558:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767798.635686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767798.635707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767798.635719:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767798.635733:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3527 00000100:00080000:0.0:1775767798.635758:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767798.635772:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767798.635812:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861504896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767798.635841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767798.635844:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767798.635846:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767798.635848:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3527 00000100:00080000:0.0:1775767798.635851:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767798.635854:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767798.635861:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767798.635870:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861505024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767798.635878:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767798.635881:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767798.635882:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767798.635885:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3527 00000100:00080000:0.0:1775767798.635886:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767798.635889:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767798.635896:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861504896:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767798.635897:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767798.635925:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861505152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767798.635986:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767798.636044:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767798/real 1775767798] req@ffffa0e9d09dd880 x1862023861504896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767798.636064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861504896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767798.636075:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767798.636080:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767798.636084:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861504896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767798.636092:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767798.636115:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767798.636128:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767798/real 1775767798] req@ffffa0e9d09df800 x1862023861505024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767798.636141:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861505024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767798.636162:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767798.636166:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767798.636170:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767798.636211:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767798.636244:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767798.636256:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767798/real 1775767798] req@ffffa0e9d09dc700 x1862023861505152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767798.636268:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861505152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767798.636275:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767798.636278:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767798.636282:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505152:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767799.211648:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767799.467598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767800.235642:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767800.491610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767801.259658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767801.515601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767802.283674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767802.539624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767803.307631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767803.563600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767803.755634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767803.755646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767803.755650:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767803.755658:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3532 00000100:00080000:0.0:1775767803.755668:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767803.755673:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767803.755718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861505280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767803.755749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767803.755754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767803.755756:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767803.755760:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3532 00000100:00080000:0.0:1775767803.755765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767803.755769:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767803.755778:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767803.755791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861505408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767803.755799:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767803.755802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767803.755803:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767803.755805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861505280:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767803.755823:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3532 00000100:00080000:0.0:1775767803.755827:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767803.755829:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767803.755834:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767803.755843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861505536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767803.755880:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767803.755915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffffa0e9d09ded80 x1862023861505280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767803.755929:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861505280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767803.755937:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767803.755940:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767803.755948:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861505280:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767803.755976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767803.755995:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767803.756004:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffffa0e9d09dfb80 x1862023861505408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767803.756012:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861505408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767803.756017:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767803.756020:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767803.756022:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505408:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767803.756025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767803.756036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767803.756042:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffffa0e9d09df100 x1862023861505536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767803.756049:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861505536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767803.756054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767803.756056:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767803.756057:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505536:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775767804.331678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767804.587590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767805.355668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767805.611575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767806.379621:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767806.635592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775767807.403664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767807.659580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767808.427666:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767808.683610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767808.875650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767808.875663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767808.875668:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767808.875679:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3537 00000100:00080000:3.0:1775767808.875690:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767808.875698:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767808.875757:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861505664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767808.875812:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767808.875816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767808.875819:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767808.875824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3537 00000100:00080000:3.0:1775767808.875829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767808.875835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767808.875847:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767808.875862:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861505664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767808.875864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861505792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767808.875880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767808.875884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767808.875886:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767808.875891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3537 00000100:00080000:3.0:1775767808.875896:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767808.875900:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767808.875906:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767808.875919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861505920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767808.875936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767808.875983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffffa0e9d0543800 x1862023861505664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767808.875998:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861505664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767808.876006:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767808.876009:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767808.876012:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861505664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767808.876033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767808.876050:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767808.876059:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffffa0e9d0540700 x1862023861505792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767808.876067:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861505792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767808.876072:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767808.876074:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767808.876076:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505792:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767808.876079:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767808.876090:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767808.876095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffffa0e9d0540e00 x1862023861505920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767808.876114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861505920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767808.876119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767808.876121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767808.876123:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861505920:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767809.451700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767809.707584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767810.475647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767810.731587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767811.499637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767811.755586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767812.523619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767812.779572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767813.547699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767813.803608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767813.995648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767813.995660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767813.995665:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767813.995676:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3542 00000100:00080000:2.0:1775767813.995688:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767813.995696:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767813.995738:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861506048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767813.995772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767813.995794:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767813.995796:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767813.995801:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3542 00000100:00080000:2.0:1775767813.995805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767813.995809:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767813.995818:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767813.995822:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861506048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775767813.995832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861506176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767813.995842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767813.995845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767813.995846:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767813.995849:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3542 00000100:00080000:2.0:1775767813.995852:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767813.995854:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767813.995857:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767813.995865:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861506304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767813.995882:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767813.995920:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffffa0e9d0710e00 x1862023861506048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767813.995934:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861506048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767813.995942:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767813.995945:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767813.995948:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861506048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767813.995981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767813.996005:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767813.996024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffffa0e9d0712680 x1862023861506176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767813.996033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861506176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767813.996049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767813.996051:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767813.996054:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767813.996058:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767813.996074:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767813.996080:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffffa0e9d0711500 x1862023861506304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767813.996088:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861506304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767813.996092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767813.996094:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767813.996096:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506304:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767814.571641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767814.827591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767815.595684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767815.851626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767816.619700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767816.875603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767817.643713:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767817.899595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767818.667634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767818.923621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767819.115794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767819.115804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767819.115807:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767819.115818:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3547 00000100:00080000:0.0:1775767819.115830:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767819.115838:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767819.115880:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861506432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767819.115905:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767819.115908:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767819.115909:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767819.115912:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3547 00000100:00080000:0.0:1775767819.115915:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767819.115918:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767819.115944:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767819.115951:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861506432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767819.115957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861506560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767819.115966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767819.115968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767819.115970:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767819.115973:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3547 00000100:00080000:0.0:1775767819.115975:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767819.115978:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767819.115982:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767819.115990:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861506688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767819.116011:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767819.116059:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767819/real 1775767819] req@ffffa0e9d09dce00 x1862023861506432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767819.116080:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861506432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767819.116092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767819.116096:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767819.116108:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861506432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767819.116128:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767819.116161:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767819.116172:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767819/real 1775767819] req@ffffa0e9d09dc380 x1862023861506560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767819.116181:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861506560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767819.116186:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767819.116188:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767819.116190:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506560:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767819.116195:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767819.116206:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767819.116212:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767819/real 1775767819] req@ffffa0e9d09dc000 x1862023861506688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767819.116219:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861506688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767819.116224:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767819.116225:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767819.116227:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506688:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767819.691677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767819.947626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767820.715661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767820.971599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767821.739652:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767821.995604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767822.763639:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767823.019602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767823.787641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767824.043609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767824.235585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767824.235595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767824.235600:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767824.235609:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3552 00000100:00080000:2.0:1775767824.235619:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767824.235625:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767824.235662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861506816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767824.235687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767824.235690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767824.235692:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767824.235695:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3552 00000100:00080000:2.0:1775767824.235699:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767824.235703:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767824.235719:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767824.235734:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861506944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767824.235747:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767824.235764:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861506816:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767824.235775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767824.235777:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767824.235783:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3552 00000100:00080000:2.0:1775767824.235788:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767824.235792:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767824.235799:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767824.235814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861507072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767824.235868:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767824.235902:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507072:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767824.811628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767825.067587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767825.835629:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767826.091577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767826.859705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767827.115596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775767827.307828:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775767827.307876:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767827.307895:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767827.307941:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767827.307967:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775767827.308022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767827] req@ffffa0e9d0713480 x1862023861506944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767827.308082:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861506944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767827.308095:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767827.308101:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767827.308105:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861506944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767827.308112:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767827] req@ffffa0e9d0711880 x1862023861507072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767827.308122:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861507072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767827.308128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767827.308131:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767827.308133:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507072:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767827.308138:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767827] req@ffffa0e9d0712a00 x1862023861506816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767827.308146:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861506816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767827.308150:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767827.308152:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767827.308178:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861506816:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775767827.883679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767828.139590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767828.907692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767829.163579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767829.355631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767829.355644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767829.355648:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767829.355656:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3557 00000100:00080000:1.0:1775767829.355666:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767829.355672:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767829.355714:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861507200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767829.355750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767829.355753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767829.355755:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767829.355760:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3557 00000100:00080000:1.0:1775767829.355763:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767829.355768:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767829.355780:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767829.355794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861507328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767829.355805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767829.355806:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861507200:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767829.355812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767829.355816:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767829.355821:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3557 00000100:00080000:1.0:1775767829.355825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767829.355850:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767829.355856:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767829.355881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861507456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767829.355886:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767829.355936:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffffa0e9c8dee300 x1862023861507200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767829.355981:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861507200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767829.355993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767829.355998:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767829.356010:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861507200:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767829.356050:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767829.356084:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767829.356098:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffffa0e9c8dec000 x1862023861507328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767829.356111:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861507328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767829.356120:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767829.356124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767829.356128:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507328:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767829.356134:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767829.356157:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767829.356168:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffffa0e9c8ded180 x1862023861507456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767829.356179:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861507456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767829.356187:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767829.356190:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767829.356193:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507456:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767829.931666:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767830.187567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767830.955733:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767831.211619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767831.979682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767832.235583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767833.003688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767833.259602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767834.027651:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767834.283561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767834.475593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767834.475604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767834.475624:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767834.475633:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3562 00000100:00080000:1.0:1775767834.475643:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767834.475649:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767834.475706:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861507584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767834.475735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767834.475739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767834.475740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767834.475744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3562 00000100:00080000:1.0:1775767834.475747:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767834.475750:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767834.475759:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767834.475769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861507712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767834.475795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767834.475798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767834.475804:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861507584:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767834.475818:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767834.475822:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3562 00000100:00080000:1.0:1775767834.475826:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767834.475829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767834.475833:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767834.475859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861507840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767834.475887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767834.475923:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767834/real 1775767834] req@ffffa0e9c8ded180 x1862023861507584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767834.475937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861507584/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767834.475945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767834.475948:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767834.475951:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861507584:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767834.475974:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767834.476093:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767834.476111:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767834/real 1775767834] req@ffffa0e9c8dec000 x1862023861507712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767834.476124:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861507712/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767834.476133:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767834.476136:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767834.476140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507712:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767834.476147:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767834.476163:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767834.476171:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767834/real 1775767834] req@ffffa0e9c8dee300 x1862023861507840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767834.476195:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861507840/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767834.476202:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767834.476205:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767834.476207:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861507840:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767835.051743:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767835.307603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767836.075639:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767836.331602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767837.099665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767837.355610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767838.123647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767838.379592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767839.147694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767839.403567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767839.595573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767839.595582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767839.595586:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767839.595595:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3568 00000100:00080000:1.0:1775767839.595604:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767839.595609:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767839.595646:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861507968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767839.595669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767839.595672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767839.595673:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767839.595676:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3568 00000100:00080000:1.0:1775767839.595678:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767839.595681:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767839.595688:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767839.595697:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861508096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767839.595723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767839.595726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767839.595728:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767839.595731:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3568 00000100:00080000:1.0:1775767839.595734:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767839.595737:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767839.595740:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767839.595750:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861507968:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767839.595753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861508224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767839.595812:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767839.595859:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767839/real 1775767839] req@ffffa0e9c8dec380 x1862023861507968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767839.595873:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861507968/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767839.595881:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767839.595885:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767839.595889:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861507968:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767839.595894:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767839.595908:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767839.595915:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767839/real 1775767839] req@ffffa0e9c8deed80 x1862023861508096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767839.595923:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861508096/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767839.595951:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767839.595954:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767839.595957:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508096:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767839.595976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508224:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767839.595991:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767839.595998:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767839/real 1775767839] req@ffffa0e9c8ded500 x1862023861508224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767839.596009:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861508224/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767839.596018:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767839.596021:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767839.596024:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508224:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767840.171612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767840.427583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767841.195678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767841.451582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767842.219762:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767842.475651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767843.243672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767843.499620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767844.267704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767844.523597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767844.715731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767844.715743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767844.715750:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767844.715780:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3573 00000100:00080000:0.0:1775767844.715800:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767844.715812:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767844.715927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861508352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767844.715965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767844.715986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767844.715989:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767844.715995:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3573 00000100:00080000:0.0:1775767844.716000:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767844.716006:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767844.716019:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767844.716039:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861508352:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767844.716040:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861508480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767844.716085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767844.716092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767844.716095:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767844.716104:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3573 00000100:00080000:0.0:1775767844.716133:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767844.716144:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775767844.716146:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767844.716161:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767844.716192:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861508608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767844.716204:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767844/real 1775767844] req@ffffa0e9d09de300 x1862023861508352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767844.716287:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861508352/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767844.716301:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767844.716307:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767844.716313:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861508352:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767844.716340:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767844.716362:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767844.716375:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767844/real 1775767844] req@ffffa0e9d09dea00 x1862023861508480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767844.716404:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861508480/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767844.716412:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767844.716415:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767844.716419:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508480:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767844.716425:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767844.716444:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767844.716453:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767844/real 1775767844] req@ffffa0e9d09df480 x1862023861508608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767844.716464:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861508608/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767844.716471:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767844.716474:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767844.716477:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508608:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767845.291657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767845.547601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767846.315657:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767846.571607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767847.339689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767847.595633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767848.363738:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767848.619637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767849.387647:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767849.643602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767849.835632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767849.835645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767849.835650:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767849.835662:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3578 00000100:00080000:1.0:1775767849.835674:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767849.835681:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767849.835719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861508736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767849.835780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767849.835784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767849.835786:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767849.835796:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3578 00000100:00080000:1.0:1775767849.835800:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767849.835804:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767849.835811:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861508736:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775767849.835815:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767849.835833:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861508864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767849.835862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767849.835866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767849.835868:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767849.835873:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3578 00000100:00080000:1.0:1775767849.835878:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767849.835881:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775767849.835886:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775767849.835886:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767849.835899:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861508992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767849.835927:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767849/real 1775767849] req@ffffa0e9c8def800 x1862023861508736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767849.835954:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861508736/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767849.835962:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767849.835965:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767849.835968:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861508736:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767849.835986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767849.836002:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767849.836010:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767849/real 1775767849] req@ffffa0e9c8def480 x1862023861508864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767849.836018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861508864/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767849.836023:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767849.836025:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767849.836027:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508864:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767849.836031:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767849.836040:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767849.836045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767849/real 1775767849] req@ffffa0e9c8dedc00 x1862023861508992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767849.836052:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861508992/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767849.836065:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767849.836067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767849.836069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861508992:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767850.411654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767850.667609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767851.435669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767851.691588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767852.459702:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767852.715620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767853.483653:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767853.739586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767854.507645:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767854.763583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767854.955610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767854.955620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767854.955625:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767854.955634:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3583 00000100:00080000:1.0:1775767854.955670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767854.955676:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767854.955709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861509120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767854.955730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767854.955733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767854.955734:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767854.955738:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3583 00000100:00080000:1.0:1775767854.955745:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767854.955749:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767854.955757:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767854.955758:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861509120:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767854.955768:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861509248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767854.955779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767854.955783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767854.955812:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767854.955818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3583 00000100:00080000:1.0:1775767854.955825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767854.955829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775767854.955834:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775767854.955837:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767854.955852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861509376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767854.955877:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffffa0e9c8dee680 x1862023861509120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767854.955891:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861509120/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767854.955899:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767854.955903:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767854.955906:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861509120:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767854.955925:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767854.955940:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767854.955948:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffffa0e9c8ded880 x1862023861509248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767854.955964:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861509248/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767854.955970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767854.955978:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767854.955980:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509248:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767854.955985:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767854.956000:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767854.956007:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffffa0e9c8deea00 x1862023861509376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767854.956019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861509376/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767854.956027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767854.956030:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767854.956033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509376:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767855.531633:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767855.787601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767856.555692:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767856.811617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767857.579627:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767857.835630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767858.603736:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767858.859611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767859.627622:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767859.883735:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767860.075614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767860.075624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767860.075628:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767860.075637:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3588 00000100:00080000:0.0:1775767860.075647:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767860.075653:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767860.075692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861509504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767860.075726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767860.075730:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767860.075731:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767860.075764:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861509504:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775767860.075769:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3588 00000100:00080000:0.0:1775767860.075775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767860.075779:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767860.075788:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767860.075801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861509632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767860.075809:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767860.075812:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767860.075813:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767860.075816:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3588 00000100:00080000:0.0:1775767860.075822:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767860.075824:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767860.075828:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767860.075837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861509760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767860.075837:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767860.075887:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767860/real 1775767860] req@ffffa0e9d09df480 x1862023861509504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767860.075906:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861509504/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767860.075914:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767860.075918:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767860.075921:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861509504:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767860.075940:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767860.075958:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767860.075967:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767860/real 1775767860] req@ffffa0e9d09dea00 x1862023861509632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767860.075979:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861509632/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767860.075987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767860.075990:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767860.075994:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509632:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767860.076008:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767860.076028:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767860.076037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767860/real 1775767860] req@ffffa0e9d09de300 x1862023861509760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767860.076054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861509760/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767860.076060:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767860.076062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767860.076064:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861509760:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767860.651671:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767860.907578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767861.675704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767861.931587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767862.699696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767862.955610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767863.723727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767863.979584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767864.747671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767865.003592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767865.195544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767865.195554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767865.195559:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767865.195569:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3593 00000100:00080000:0.0:1775767865.195581:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767865.195587:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767865.195628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861509888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767865.195668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767865.195671:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767865.195673:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767865.195676:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3593 00000100:00080000:0.0:1775767865.195679:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767865.195682:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767865.195704:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767865.195718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861510016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767865.195728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767865.195731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767865.195733:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767865.195736:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3593 00000100:00080000:0.0:1775767865.195740:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767865.195743:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767865.195747:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767865.195755:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861510144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767865.195765:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861509888:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767865.195847:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767865.195891:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffffa0e9d09dc000 x1862023861509888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767865.195912:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861509888/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767865.195920:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767865.195924:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767865.195927:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861509888:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767865.195933:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510016:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767865.195951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767865.195959:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffffa0e9d09dc380 x1862023861510016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767865.195967:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861510016/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767865.195989:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767865.195992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767865.195994:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510016:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767865.196013:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767865.196027:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767865.196034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffffa0e9d09dce00 x1862023861510144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767865.196041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861510144/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767865.196046:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767865.196048:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767865.196050:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510144:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767865.771640:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767866.027556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767866.795622:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767867.051596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767867.819707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767868.075573:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767868.843621:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767869.099610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767869.867736:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767870.123618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767870.315626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767870.315634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767870.315651:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767870.315677:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3598 00000100:00080000:0.0:1775767870.315688:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767870.315695:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767870.315736:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861510272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767870.315761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767870.315764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767870.315766:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767870.315769:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3598 00000100:00080000:0.0:1775767870.315771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767870.315774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767870.315783:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767870.315792:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861510400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767870.315799:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767870.315802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767870.315803:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767870.315806:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3598 00000100:00080000:0.0:1775767870.315808:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767870.315810:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767870.315813:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767870.315821:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861510528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767870.315821:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861510272:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767870.315915:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767870.315962:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffffa0e9d09df100 x1862023861510272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767870.315980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861510272/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767870.315989:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767870.315992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767870.316002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861510272:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767870.316008:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767870.316026:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767870.316034:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffffa0e9d09dfb80 x1862023861510400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767870.316042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861510400/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767870.316048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767870.316050:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767870.316052:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510400:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767870.316085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767870.316106:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767870.316112:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffffa0e9d09ded80 x1862023861510528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767870.316120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861510528/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767870.316126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767870.316128:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767870.316130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510528:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767870.891710:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767871.147612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767871.915697:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767872.171633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767872.939696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767873.195579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767873.963707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767874.219588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767874.987704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767875.243582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767875.435655:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767875.435664:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767875.435668:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767875.435684:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3603 00000100:00080000:3.0:1775767875.435694:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767875.435700:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767875.435740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861510656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767875.435771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767875.435780:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767875.435782:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767875.435786:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3603 00000100:00080000:3.0:1775767875.435791:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767875.435795:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767875.435805:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767875.435817:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861510656:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767875.435824:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861510784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767875.435834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767875.435836:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767875.435838:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767875.435841:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3603 00000100:00080000:3.0:1775767875.435843:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767875.435864:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767875.435869:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767875.435877:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:3.0:1775767875.435879:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861510912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767875.435938:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffffa0e9d0540a80 x1862023861510656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767875.435953:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861510656/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767875.435961:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767875.435964:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767875.435967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861510656:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767875.435986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767875.436002:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767875.436011:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffffa0e9d0542d80 x1862023861510784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767875.436019:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861510784/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767875.436024:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767875.436027:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767875.436029:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510784:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767875.436032:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510912:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767875.436042:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767875.436048:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffffa0e9d0541f80 x1862023861510912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767875.436056:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861510912/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767875.436070:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767875.436072:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767875.436074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861510912:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767876.011692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767876.267647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767877.035692:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767877.291601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767878.059669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767878.315608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767879.083688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767879.339593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767880.107644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767880.363630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767880.555618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767880.555627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767880.555631:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767880.555640:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3609 00000100:00080000:3.0:1775767880.555649:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767880.555655:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767880.555690:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861511040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767880.555719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767880.555723:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767880.555725:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767880.555729:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3609 00000100:00080000:3.0:1775767880.555734:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767880.555739:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767880.555750:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767880.555766:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861511040:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767880.555766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861511168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767880.555780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767880.555783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767880.555800:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767880.555804:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3609 00000100:00080000:3.0:1775767880.555808:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767880.555811:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767880.555816:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767880.555828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861511296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767880.555839:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767880.555877:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffffa0e9d0541180 x1862023861511040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.555891:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861511040/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767880.555899:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.555903:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767880.555906:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861511040:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767880.555926:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767880.555941:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767880.555949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffffa0e9d0543480 x1862023861511168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.555956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861511168/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767880.555970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.555972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767880.555975:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511168:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767880.555986:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767880.556001:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767880.556008:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffffa0e9d0543100 x1862023861511296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.556015:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861511296/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767880.556020:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.556022:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767880.556024:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511296:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767881.131672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767881.387638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767882.155708:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767882.411658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767883.179693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767883.435596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767884.203639:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767884.459609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767885.227680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767885.483630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767885.675619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767885.675631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767885.675637:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767885.675648:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3614 00000100:00080000:3.0:1775767885.675659:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767885.675667:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767885.675708:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861511424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767885.675734:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767885.675736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767885.675738:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767885.675741:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3614 00000100:00080000:3.0:1775767885.675743:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767885.675746:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767885.675753:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767885.675763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861511552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767885.675789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861511424:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767885.675810:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767885.675813:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767885.675814:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767885.675818:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3614 00000100:00080000:3.0:1775767885.675821:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767885.675824:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767885.675828:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767885.675839:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861511680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767885.675894:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767885.675940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffffa0e9d0543100 x1862023861511424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.675959:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861511424/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767885.675968:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.675971:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767885.675974:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861511424:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767885.675992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767885.676010:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767885.676022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffffa0e9d0543480 x1862023861511552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.676044:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861511552/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767885.676054:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.676057:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767885.676060:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511552:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767885.676066:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767885.676099:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767885.676108:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffffa0e9d0541180 x1862023861511680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.676116:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861511680/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767885.676122:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.676124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767885.676126:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511680:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767886.251644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767886.507580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767887.275639:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767887.531641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767888.299633:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767888.555577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767889.323674:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767889.579616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767890.347661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767890.603623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767890.795641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767890.795650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767890.795655:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767890.795665:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3619 00000100:00080000:0.0:1775767890.795677:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767890.795683:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767890.795727:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861511808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767890.795756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767890.795759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767890.795760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767890.795763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3619 00000100:00080000:0.0:1775767890.795766:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767890.795768:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767890.795777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767890.795785:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861511936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767890.795793:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767890.795810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767890.795812:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767890.795815:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3619 00000100:00080000:0.0:1775767890.795818:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767890.795823:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767890.795825:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861511808:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775767890.795828:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767890.795840:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861512064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767890.795940:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511936:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767890.795978:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512064:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767891.371688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767891.627598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767892.395664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767892.651590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767893.419690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767893.675590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775767893.867767:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775767893.867818:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767893.867842:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767893.867896:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775767893.867927:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775767893.867961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767893] req@ffffa0e9d09dd880 x1862023861512064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767893.868040:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861512064/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767893.868053:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767893.868059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767893.868064:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512064:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767893.868073:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767893] req@ffffa0e9d09dc700 x1862023861511808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767893.868087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861511808/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767893.868094:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767893.868098:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767893.868101:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861511808:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767893.868107:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767893] req@ffffa0e9d09df800 x1862023861511936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767893.868120:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861511936/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767893.868128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767893.868132:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767893.868135:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861511936:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767894.443632:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767894.699588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767895.467726:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767895.723640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767895.915625:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767895.915659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767895.915664:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767895.915672:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3624 00000100:00080000:1.0:1775767895.915681:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767895.915687:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767895.915718:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861512192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767895.915742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767895.915745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767895.915747:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767895.915750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3624 00000100:00080000:1.0:1775767895.915753:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767895.915756:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767895.915765:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767895.915774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861512320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767895.915779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861512192:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767895.915786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767895.915789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767895.915790:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767895.915793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3624 00000100:00080000:1.0:1775767895.915795:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767895.915798:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767895.915801:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767895.915808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861512448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767895.915845:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767895.915894:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffffa0e9c8deca80 x1862023861512192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767895.915911:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861512192/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767895.915919:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767895.915922:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767895.915925:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861512192:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767895.915952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767895.915976:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767895.915989:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffffa0e9c8dedf80 x1862023861512320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767895.916017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861512320/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767895.916027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767895.916031:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767895.916034:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512320:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767895.916040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767895.916063:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767895.916074:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffffa0e9c8dece00 x1862023861512448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767895.916085:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861512448/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767895.916092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767895.916095:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767895.916098:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512448:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767896.491679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767896.747571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767897.515676:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767897.771649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767898.539703:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767898.795598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767899.563718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767899.819614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767900.587676:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767900.843631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767901.035638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767901.035650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767901.035673:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767901.035690:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3629 00000100:00080000:0.0:1775767901.035704:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767901.035712:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767901.035757:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861512576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767901.035794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767901.035798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767901.035800:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767901.035806:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3629 00000100:00080000:0.0:1775767901.035811:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767901.035816:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767901.035829:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767901.035829:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861512576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767901.035847:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861512704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767901.035861:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767901.035865:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767901.035868:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767901.035872:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3629 00000100:00080000:0.0:1775767901.035891:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767901.035896:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767901.035903:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767901.035903:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767901.035918:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861512832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767901.035946:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767901/real 1775767901] req@ffffa0e9d09dd500 x1862023861512576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767901.035963:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861512576/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767901.035972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767901.035975:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767901.035979:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861512576:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767901.036008:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767901.036028:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767901.036037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767901/real 1775767901] req@ffffa0e9d09dd180 x1862023861512704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767901.036045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861512704/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767901.036050:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767901.036053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767901.036055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512704:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767901.036058:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767901.036069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767901.036082:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767901/real 1775767901] req@ffffa0e9d09ddc00 x1862023861512832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767901.036090:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861512832/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767901.036096:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767901.036098:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767901.036100:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861512832:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767901.611698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767901.867611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767902.635672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767902.891639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767903.659700:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767903.915620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767904.683692:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767904.939591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767905.707695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767905.963576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767906.155641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767906.155653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767906.155659:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767906.155670:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3634 00000100:00080000:3.0:1775767906.155689:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767906.155699:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767906.155746:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861512960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767906.155771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767906.155774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767906.155776:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767906.155779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3634 00000100:00080000:3.0:1775767906.155783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767906.155786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767906.155809:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767906.155819:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861513088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767906.155831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767906.155833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767906.155835:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767906.155838:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3634 00000100:00080000:3.0:1775767906.155840:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767906.155843:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767906.155846:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767906.155852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861513216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767906.155882:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861512960:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775767906.155954:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767906.155997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767906/real 1775767906] req@ffffa0e9d0541f80 x1862023861512960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767906.156018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861512960/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767906.156031:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767906.156035:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767906.156040:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861512960:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767906.156048:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767906.156071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767906.156084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767906/real 1775767906] req@ffffa0e9d0542d80 x1862023861513088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767906.156098:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861513088/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767906.156121:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767906.156125:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767906.156129:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513088:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767906.156136:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767906.156159:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767906.156171:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767906/real 1775767906] req@ffffa0e9d0540a80 x1862023861513216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767906.156184:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861513216/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767906.156192:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767906.156196:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767906.156200:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513216:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767906.731663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767906.987621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767907.755664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767908.011604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767908.779656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767909.035657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767909.803677:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767910.059582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767910.827660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767911.083616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767911.275620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767911.275632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767911.275637:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767911.275649:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3639 00000100:00080000:3.0:1775767911.275660:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767911.275667:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767911.275709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861513344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767911.275741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767911.275745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767911.275747:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767911.275751:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3639 00000100:00080000:3.0:1775767911.275774:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767911.275780:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767911.275789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861513344:192.168.202.104@tcp:250:kworker.0 00000008:00100000:3.0:1775767911.275791:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767911.275812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861513472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767911.275825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767911.275830:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767911.275832:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767911.275836:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3639 00000100:00080000:3.0:1775767911.275840:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767911.275844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767911.275850:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767911.275858:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:3.0:1775767911.275864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861513600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767911.275906:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767911/real 1775767911] req@ffffa0e9d0540e00 x1862023861513344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767911.275924:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861513344/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767911.275931:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767911.275935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767911.275945:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861513344:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767911.275965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767911.275984:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767911.276003:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767911/real 1775767911] req@ffffa0e9d0540700 x1862023861513472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767911.276011:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861513472/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767911.276017:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767911.276028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767911.276030:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513472:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767911.276034:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767911.276045:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767911.276052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767911/real 1775767911] req@ffffa0e9d0543800 x1862023861513600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767911.276071:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861513600/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767911.276076:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767911.276079:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767911.276081:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513600:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767911.851706:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767912.107619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767912.875708:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767913.131614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767913.899740:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767914.155607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767914.923696:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767915.179611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767915.947670:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767916.203570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767916.395578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767916.395588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767916.395593:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767916.395605:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3644 00000100:00080000:3.0:1775767916.395618:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767916.395625:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767916.395672:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861513728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767916.395707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767916.395727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767916.395729:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767916.395735:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3644 00000100:00080000:3.0:1775767916.395741:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767916.395746:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767916.395756:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767916.395759:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861513728:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767916.395770:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861513856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767916.395781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767916.395784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767916.395785:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767916.395788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3644 00000100:00080000:3.0:1775767916.395792:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767916.395794:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767916.395798:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767916.395815:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861513984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767916.395832:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767916.395884:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767916/real 1775767916] req@ffffa0e9d0542300 x1862023861513728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767916.395907:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861513728/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767916.395930:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767916.395935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767916.395939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861513728:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767916.395965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767916.395985:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767916.395995:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767916/real 1775767916] req@ffffa0e9d0540380 x1862023861513856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767916.396015:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861513856/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767916.396021:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767916.396023:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767916.396025:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513856:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767916.396029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767916.396039:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767916.396045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767916/real 1775767916] req@ffffa0e9d0541c00 x1862023861513984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767916.396056:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861513984/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767916.396077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767916.396081:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767916.396084:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861513984:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767916.971679:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767917.227577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767917.995649:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767918.251599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767919.019680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767919.275616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767920.043725:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767920.299591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767921.067655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767921.323609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767921.515605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767921.515614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767921.515618:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767921.515627:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3650 00000100:00080000:1.0:1775767921.515637:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767921.515643:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767921.515680:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861514112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767921.515705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767921.515708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767921.515709:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767921.515712:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3650 00000100:00080000:1.0:1775767921.515714:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767921.515716:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767921.515725:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767921.515734:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861514240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767921.515747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767921.515750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767921.515758:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861514112:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767921.515772:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767921.515778:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3650 00000100:00080000:1.0:1775767921.515783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767921.515788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767921.515795:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767921.515808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861514368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767921.515825:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767921.515874:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767921/real 1775767921] req@ffffa0e9c8dec700 x1862023861514112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767921.515902:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861514112/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767921.515913:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767921.515917:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767921.515921:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861514112:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767921.515954:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767921.515977:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767921.515989:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767921/real 1775767921] req@ffffa0e9c8defb80 x1862023861514240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767921.516003:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861514240/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767921.516013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767921.516016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767921.516019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514240:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767921.516025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767921.516042:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767921.516050:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767921/real 1775767921] req@ffffa0e9c8def100 x1862023861514368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767921.516062:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861514368/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767921.516081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767921.516085:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767921.516088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514368:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767922.091697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767922.347591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767923.115656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767923.371594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767924.139672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767924.395630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767925.163632:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767925.419605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767926.187722:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767926.443574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767926.635615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767926.635634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767926.635640:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767926.635650:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3655 00000100:00080000:1.0:1775767926.635662:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767926.635668:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767926.635710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861514496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767926.635743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767926.635747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767926.635749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767926.635754:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3655 00000100:00080000:1.0:1775767926.635758:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767926.635762:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767926.635775:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767926.635792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861514624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767926.635799:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861514496:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767926.635830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767926.635835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767926.635836:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767926.635841:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3655 00000100:00080000:1.0:1775767926.635846:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767926.635850:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767926.635857:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767926.635871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861514752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767926.635884:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767926.635934:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767926/real 1775767926] req@ffffa0e9c8def100 x1862023861514496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767926.635952:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861514496/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767926.635960:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767926.635964:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767926.635967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861514496:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767926.635992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767926.636018:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767926.636032:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767926/real 1775767926] req@ffffa0e9c8defb80 x1862023861514624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767926.636057:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861514624/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767926.636064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767926.636066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767926.636069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514624:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767926.636073:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767926.636117:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767926.636131:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767926/real 1775767926] req@ffffa0e9c8dec700 x1862023861514752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767926.636145:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861514752/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767926.636153:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767926.636156:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767926.636159:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861514752:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767927.211634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767927.467598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767928.235631:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767928.491587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767929.259634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767929.515571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767930.283700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767930.539601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767931.307665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767931.563552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767931.755601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767931.755613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767931.755618:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767931.755628:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3660 00000100:00080000:1.0:1775767931.755637:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767931.755644:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767931.755686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861514880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767931.755720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767931.755732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767931.755733:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767931.755736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3660 00000100:00080000:1.0:1775767931.755739:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767931.755742:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767931.755767:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767931.755780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861515008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767931.755781:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861514880:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767931.755795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767931.755803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767931.755805:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767931.755808:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3660 00000100:00080000:1.0:1775767931.755810:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767931.755813:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767931.755816:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767931.755824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861515136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767931.755864:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767931.755916:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffffa0e9c8dece00 x1862023861514880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767931.755937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861514880/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767931.755949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767931.755954:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767931.755958:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861514880:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767931.755980:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515008:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767931.756002:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767931.756013:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffffa0e9c8dedf80 x1862023861515008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767931.756025:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861515008/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767931.756034:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767931.756037:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767931.756040:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515008:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767931.756077:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767931.756095:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767931.756106:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffffa0e9c8deca80 x1862023861515136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767931.756133:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861515136/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767931.756142:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767931.756145:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767931.756149:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515136:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767932.331683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767932.587624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767933.355681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767933.611600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767934.379682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767934.635617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767935.403603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767935.659578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767936.427633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767936.683579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767936.875600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767936.875611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767936.875616:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767936.875626:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3665 00000100:00080000:3.0:1775767936.875636:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767936.875642:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767936.875681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861515264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767936.875733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767936.875736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767936.875738:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767936.875741:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3665 00000100:00080000:3.0:1775767936.875745:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767936.875748:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767936.875757:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767936.875769:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861515264:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775767936.875772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861515392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767936.875782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767936.875784:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767936.875786:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767936.875789:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3665 00000100:00080000:3.0:1775767936.875791:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767936.875793:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767936.875796:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767936.875804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861515520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767936.875829:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767936.875866:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffffa0e9d0540000 x1862023861515264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767936.875879:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861515264/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767936.875887:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767936.875891:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767936.875894:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861515264:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767936.875910:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767936.875924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767936.875940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffffa0e9d0542a00 x1862023861515392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767936.875948:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861515392/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767936.875954:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767936.875956:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767936.875958:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515392:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767936.875962:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767936.875973:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767936.875979:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffffa0e9d0543b80 x1862023861515520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767936.875986:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861515520/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767936.875991:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767936.875992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767936.876018:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515520:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767937.451634:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767937.707586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767938.475699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767938.731588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767939.499710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767939.755577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767940.523655:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767940.779593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767941.547704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767941.803605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767941.995643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767941.995652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767941.995658:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767941.995667:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3670 00000100:00080000:3.0:1775767941.995699:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767941.995706:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767941.995744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861515648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767941.995771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767941.995774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767941.995776:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767941.995779:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3670 00000100:00080000:3.0:1775767941.995781:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767941.995783:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767941.995793:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767941.995800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861515776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767941.995808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767941.995811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767941.995813:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767941.995816:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3670 00000100:00080000:3.0:1775767941.995817:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767941.995820:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767941.995822:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767941.995828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861515904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767941.995837:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861515648:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767941.995923:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767941.996001:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffffa0e9d0542680 x1862023861515648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.996022:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861515648/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767941.996034:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.996038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767941.996042:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861515648:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767941.996050:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767941.996071:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767941.996084:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffffa0e9d0541500 x1862023861515776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.996096:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861515776/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767941.996105:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.996108:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767941.996111:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515776:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767941.996140:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515904:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767941.996162:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767941.996175:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffffa0e9d0541880 x1862023861515904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.996200:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861515904/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767941.996206:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.996209:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767941.996211:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861515904:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767942.571654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767942.827572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767943.595692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767943.851581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767944.619650:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767944.875568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767945.643684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767945.899577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767946.667680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767946.923566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767947.115594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767947.115623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767947.115627:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767947.115635:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3675 00000100:00080000:3.0:1775767947.115645:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767947.115651:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767947.115683:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861516032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767947.115708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767947.115712:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767947.115714:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767947.115718:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3675 00000100:00080000:3.0:1775767947.115722:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767947.115727:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767947.115737:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767947.115751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861516160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767947.115761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767947.115765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767947.115768:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767947.115773:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3675 00000100:00080000:3.0:1775767947.115778:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767947.115783:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767947.115788:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767947.115805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861516288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767947.115809:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861516032:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767947.115917:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767947.115991:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767947/real 1775767947] req@ffffa0e9d0541880 x1862023861516032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767947.116012:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861516032/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767947.116022:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767947.116026:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767947.116030:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861516032:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767947.116037:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767947.116088:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767947.116099:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767947/real 1775767947] req@ffffa0e9d0541500 x1862023861516160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767947.116109:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861516160/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767947.116116:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767947.116119:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767947.116122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516160:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767947.116148:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767947.116161:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767947.116168:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767947/real 1775767947] req@ffffa0e9d0542680 x1862023861516288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767947.116185:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861516288/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767947.116191:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767947.116194:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767947.116197:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516288:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767947.691688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767947.947641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767948.715676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767948.971608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767949.739726:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767949.995606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767950.763672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767951.019580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767951.787719:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767952.043597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767952.235621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767952.235630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767952.235635:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767952.235643:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3680 00000100:00080000:2.0:1775767952.235653:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767952.235658:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767952.235695:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861516416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767952.235720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767952.235722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767952.235724:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767952.235726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3680 00000100:00080000:2.0:1775767952.235729:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767952.235731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767952.235739:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767952.235747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861516544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767952.235790:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861516416:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767952.235796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767952.235800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767952.235801:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767952.235805:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3680 00000100:00080000:2.0:1775767952.235808:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767952.235812:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767952.235817:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767952.235835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861516672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767952.235880:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767952.235930:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffffa0e9d0713100 x1862023861516416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767952.235949:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861516416/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767952.235962:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767952.235967:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767952.235973:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861516416:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767952.236002:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767952.236026:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767952.236037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffffa0e9d0712300 x1862023861516544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767952.236046:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861516544/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767952.236052:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767952.236054:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767952.236057:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516544:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767952.236061:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767952.236073:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767952.236082:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffffa0e9d0712a00 x1862023861516672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767952.236117:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861516672/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767952.236126:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767952.236130:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767952.236133:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516672:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767952.811694:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767953.067624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767953.835707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767954.091653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767954.859656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767955.115615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767955.883668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767956.139701:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767956.907672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767957.163663:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767957.355622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767957.355631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767957.355635:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767957.355644:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3685 00000100:00080000:3.0:1775767957.355654:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767957.355659:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775767957.355706:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861516800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767957.355740:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767957.355745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767957.355747:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767957.355757:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3685 00000100:00080000:3.0:1775767957.355760:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767957.355764:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767957.355773:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767957.355788:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861516928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767957.355813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767957.355817:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767957.355820:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767957.355826:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3685 00000100:00080000:3.0:1775767957.355831:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767957.355840:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861516800:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775767957.355841:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775767957.355867:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767957.355881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861517056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767957.355963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516928:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767957.356009:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517056:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767957.931629:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767958.187683:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767958.955737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767959.211650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767959.979684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767960.235632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775767960.427786:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775767960.427832:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767960.427846:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767960.427884:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775767960.427895:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775767960.427960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767960] req@ffffa0e9d0540000 x1862023861517056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767960.427988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861517056/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767960.428023:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767960.428029:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767960.428035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517056:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767960.428045:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767960] req@ffffa0e9d0543b80 x1862023861516800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767960.428065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861516800/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767960.428073:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767960.428077:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767960.428079:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861516800:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767960.428087:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767960] req@ffffa0e9d0542a00 x1862023861516928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767960.428098:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861516928/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767960.428107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767960.428112:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767960.428116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861516928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767961.003690:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767961.259567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767962.027616:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767962.283657:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767962.475619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767962.475629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767962.475637:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767962.475647:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3690 00000100:00080000:1.0:1775767962.475656:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767962.475662:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767962.475697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861517184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767962.475723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767962.475729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767962.475730:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767962.475733:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3690 00000100:00080000:1.0:1775767962.475764:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767962.475768:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767962.475775:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861517184:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775767962.475777:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767962.475796:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861517312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767962.475809:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767962.475813:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767962.475815:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767962.475819:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3690 00000100:00080000:1.0:1775767962.475826:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767962.475831:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775767962.475837:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775767962.475837:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767962.475853:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861517440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767962.475896:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffffa0e9c8deea00 x1862023861517184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767962.475917:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861517184/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767962.475928:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767962.475933:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767962.475938:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861517184:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767962.475963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767962.476000:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767962.476014:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffffa0e9c8ded880 x1862023861517312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767962.476026:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861517312/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767962.476035:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767962.476038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767962.476041:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517312:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767962.476047:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767962.476072:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767962.476083:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffffa0e9c8dee680 x1862023861517440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767962.476097:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861517440/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767962.476117:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767962.476122:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767962.476125:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517440:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767963.051682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767963.307616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767964.075667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767964.331617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767965.099696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767965.355616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767966.123707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767966.379576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767967.147745:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767967.403615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767967.595612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767967.595624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767967.595630:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767967.595640:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3696 00000100:00080000:1.0:1775767967.595653:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767967.595661:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767967.595724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861517568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767967.595762:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767967.595779:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767967.595781:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767967.595786:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3696 00000100:00080000:1.0:1775767967.595790:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767967.595794:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767967.595804:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767967.595817:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861517568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767967.595818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861517696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767967.595832:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767967.595834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767967.595835:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767967.595838:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3696 00000100:00080000:1.0:1775767967.595841:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767967.595844:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767967.595848:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767967.595857:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861517824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767967.595902:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767967.595947:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767967] req@ffffa0e9c8dedc00 x1862023861517568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767967.595965:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861517568/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767967.595975:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767967.595978:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767967.595981:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861517568:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767967.596001:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767967.596018:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767967.596027:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767967] req@ffffa0e9c8def480 x1862023861517696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767967.596036:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861517696/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767967.596041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767967.596044:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767967.596052:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517696:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767967.596057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767967.596072:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767967.596079:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767967] req@ffffa0e9c8def800 x1862023861517824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767967.596087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861517824/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767967.596092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767967.596094:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767967.596095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861517824:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767968.171712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767968.427583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767969.195649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767969.451618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767970.219643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767970.475596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767971.243704:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767971.499618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767972.267639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767972.523628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775767972.715571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767972.715583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767972.715588:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767972.715596:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3701 00000100:00080000:0.0:1775767972.715606:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767972.715612:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775767972.715661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861517952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767972.715688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767972.715690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767972.715692:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767972.715714:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3701 00000100:00080000:0.0:1775767972.715717:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767972.715722:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767972.715731:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767972.715731:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861517952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767972.715745:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861518080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767972.715754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767972.715757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767972.715758:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767972.715761:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3701 00000100:00080000:0.0:1775767972.715764:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767972.715767:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775767972.715770:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767972.715777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861518208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767972.715793:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767972.715830:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffffa0e9d09ddf80 x1862023861517952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767972.715844:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861517952/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767972.715862:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767972.715865:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767972.715868:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861517952:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767972.715888:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767972.715916:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767972.715925:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffffa0e9d09dca80 x1862023861518080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767972.715933:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861518080/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767972.715939:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767972.715941:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767972.715943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518080:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775767972.715947:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775767972.715958:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767972.715963:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffffa0e9d09de680 x1862023861518208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767972.715970:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861518208/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767972.715981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767972.715983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775767972.715985:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518208:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767973.291630:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767973.547615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767974.315695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767974.571651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767975.339718:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767975.595643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767976.363639:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767976.619580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767977.387713:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767977.643579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767977.835584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767977.835607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767977.835613:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767977.835624:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3706 00000100:00080000:1.0:1775767977.835635:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767977.835642:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767977.835683:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861518336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767977.835715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767977.835719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767977.835721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767977.835726:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3706 00000100:00080000:1.0:1775767977.835730:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767977.835735:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767977.835746:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767977.835749:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861518336:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767977.835761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861518464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767977.835770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767977.835773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767977.835774:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767977.835777:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3706 00000100:00080000:1.0:1775767977.835781:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767977.835784:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767977.835805:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767977.835810:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775767977.835816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861518592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767977.835849:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffffa0e9c8ded500 x1862023861518336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767977.835864:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861518336/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767977.835882:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767977.835886:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767977.835889:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861518336:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767977.835911:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767977.835936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767977.835949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffffa0e9c8deed80 x1862023861518464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767977.835963:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861518464/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767977.835972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767977.835974:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767977.835977:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518464:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767977.835981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767977.835996:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767977.836003:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffffa0e9c8dec380 x1862023861518592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767977.836010:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861518592/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767977.836015:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767977.836017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767977.836019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518592:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767978.411627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767978.667561:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767979.435698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767979.691605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767980.459737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767980.715658:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767981.483670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767981.739630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767982.507704:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767982.763765:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767982.955620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767982.955630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767982.955636:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767982.955646:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3711 00000100:00080000:1.0:1775767982.955657:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767982.955664:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767982.955707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861518720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767982.955738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767982.955741:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767982.955743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767982.955746:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3711 00000100:00080000:1.0:1775767982.955749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767982.955752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767982.955760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767982.955771:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861518848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767982.955780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767982.955783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767982.955784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767982.955787:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3711 00000100:00080000:1.0:1775767982.955808:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767982.955811:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767982.955815:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767982.955825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861518976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767982.955827:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861518720:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775767982.955919:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767982.955962:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffffa0e9c8dee300 x1862023861518720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767982.955980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861518720/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767982.955989:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767982.955992:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767982.955995:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861518720:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767982.956002:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767982.956017:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767982.956025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffffa0e9c8dec000 x1862023861518848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767982.956033:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861518848/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767982.956038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767982.956040:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767982.956042:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518848:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767982.956064:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767982.956075:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767982.956097:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffffa0e9c8ded180 x1862023861518976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767982.956105:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861518976/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767982.956111:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767982.956113:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767982.956116:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861518976:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767983.531650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767983.787600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767984.555698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767984.811584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767985.579723:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767985.835585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767986.603700:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767986.859636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767987.627700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767987.883585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767988.075630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767988.075642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767988.075648:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767988.075661:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3716 00000100:00080000:1.0:1775767988.075675:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767988.075683:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767988.075733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861519104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767988.075770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767988.075774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767988.075776:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767988.075781:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3716 00000100:00080000:1.0:1775767988.075785:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767988.075790:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767988.075804:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767988.075817:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861519104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775767988.075823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861519232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767988.075837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767988.075857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767988.075860:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767988.075866:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3716 00000100:00080000:1.0:1775767988.075882:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767988.075888:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:0.0:1775767988.075893:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:1.0:1775767988.075896:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767988.075914:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861519360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767988.075941:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767988/real 1775767988] req@ffffa0e9c8ded180 x1862023861519104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767988.075957:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861519104/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767988.075966:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767988.075969:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767988.075972:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861519104:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767988.075993:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767988.076008:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767988.076016:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767988/real 1775767988] req@ffffa0e9c8dec000 x1862023861519232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767988.076037:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861519232/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767988.076042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767988.076045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767988.076047:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519232:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767988.076050:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767988.076063:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767988.076069:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767988/real 1775767988] req@ffffa0e9c8dee300 x1862023861519360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767988.076076:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861519360/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767988.076081:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767988.076083:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767988.076085:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519360:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767988.651699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767988.907674:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767989.675664:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767989.931623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767990.699846:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767990.955603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767991.723670:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767991.979601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767992.747692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767993.003645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767993.195616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767993.195624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767993.195629:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767993.195639:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3721 00000100:00080000:2.0:1775767993.195651:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767993.195658:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775767993.195705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861519488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767993.195740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767993.195744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767993.195746:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767993.195750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3721 00000100:00080000:2.0:1775767993.195755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767993.195759:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767993.195769:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767993.195781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861519616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767993.195792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767993.195796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767993.195797:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767993.195800:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3721 00000100:00080000:2.0:1775767993.195804:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767993.195815:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861519488:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775767993.195827:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775767993.195836:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767993.195844:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861519744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767993.195905:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767993.195955:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffffa0e9d0711880 x1862023861519488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767993.195977:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861519488/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767993.195998:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767993.196001:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767993.196004:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861519488:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767993.196011:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767993.196030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767993.196042:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffffa0e9d0713480 x1862023861519616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767993.196055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861519616/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767993.196063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767993.196067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767993.196070:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519616:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767993.196100:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767993.196120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767993.196131:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffffa0e9d0711500 x1862023861519744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767993.196143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861519744/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767993.196151:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767993.196154:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767993.196158:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861519744:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775767993.771626:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767994.027593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767994.795677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767995.051601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775767995.819636:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767996.075603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767996.843690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767997.099660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767997.867654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767998.123627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775767998.315633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767998.315643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767998.315647:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767998.315656:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3726 00000100:00080000:1.0:1775767998.315665:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767998.315671:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775767998.315708:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861519872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767998.315734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767998.315737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767998.315738:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767998.315741:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3726 00000100:00080000:1.0:1775767998.315744:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767998.315746:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767998.315754:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767998.315762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861520000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767998.315781:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861519872:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775767998.315796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767998.315798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767998.315800:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767998.315804:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3726 00000100:00080000:1.0:1775767998.315807:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767998.315809:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775767998.315814:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767998.315823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861520128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767998.315852:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767998.315888:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffffa0e9c8dec380 x1862023861519872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767998.315902:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861519872/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767998.315910:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767998.315914:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767998.315917:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861519872:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775767998.315934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520000:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767998.315949:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767998.315957:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffffa0e9c8deed80 x1862023861520000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767998.315964:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861520000/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767998.315970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767998.315972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767998.315974:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520000:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775767998.315978:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775767998.315987:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775767998.316001:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffffa0e9c8ded500 x1862023861520128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767998.316009:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861520128/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767998.316014:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767998.316016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775767998.316019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520128:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775767998.891633:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767999.147597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775767999.915704:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768000.171603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768000.939681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768001.195625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768001.963689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768002.219636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768002.987668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768003.243589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768003.435575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768003.435584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768003.435589:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768003.435598:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3731 00000100:00080000:2.0:1775768003.435607:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768003.435612:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768003.435650:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861520256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768003.435692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768003.435695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768003.435697:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768003.435700:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3731 00000100:00080000:2.0:1775768003.435703:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768003.435705:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768003.435714:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768003.435725:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861520384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768003.435736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768003.435740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768003.435742:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768003.435746:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3731 00000100:00100000:0.0:1775768003.435752:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861520256:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768003.435753:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768003.435763:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768003.435769:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768003.435785:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861520512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768003.435830:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768003.435883:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffffa0e9d0712680 x1862023861520256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.435904:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861520256/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768003.435917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.435922:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768003.435927:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861520256:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768003.435935:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768003.435960:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768003.435974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffffa0e9d0710e00 x1862023861520384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.436004:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861520384/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768003.436013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.436017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768003.436021:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520384:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768003.436049:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768003.436070:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768003.436081:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffffa0e9d0710a80 x1862023861520512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.436093:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861520512/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768003.436102:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.436106:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768003.436109:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520512:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768004.011718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768004.267620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768005.035689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768005.291570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768006.059688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768006.315613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768007.083660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768007.339579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768008.107695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768008.363611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768008.555620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768008.555628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768008.555632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768008.555639:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3737 00000100:00080000:0.0:1775768008.555647:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768008.555653:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768008.555704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861520640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768008.555734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768008.555738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768008.555740:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768008.555744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3737 00000100:00080000:0.0:1775768008.555749:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768008.555753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768008.555763:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768008.555769:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861520640:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768008.555778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861520768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768008.555790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768008.555794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768008.555795:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768008.555798:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3737 00000100:00080000:0.0:1775768008.555813:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768008.555815:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768008.555819:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775768008.555821:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768008.555831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861520896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768008.555858:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffffa0e9d09de680 x1862023861520640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768008.555877:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861520640/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768008.555887:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768008.555891:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768008.555896:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861520640:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768008.555925:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768008.555942:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768008.555950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffffa0e9d09dca80 x1862023861520768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768008.555964:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861520768/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768008.555970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768008.555973:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768008.555976:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520768:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768008.555981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520896:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768008.556000:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768008.556010:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffffa0e9d09ddf80 x1862023861520896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768008.556021:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861520896/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768008.556028:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768008.556030:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768008.556032:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861520896:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768009.131700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768009.387546:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768010.155671:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768010.411638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768011.179685:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768011.435610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768012.203675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768012.459631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768013.227658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768013.483600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768013.675681:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768013.675689:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768013.675694:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768013.675710:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3742 00000100:00080000:1.0:1775768013.675721:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768013.675727:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768013.675765:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861521024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768013.675800:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768013.675802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768013.675804:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768013.675806:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3742 00000100:00080000:1.0:1775768013.675814:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768013.675818:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768013.675827:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768013.675836:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861521152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768013.675844:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861521024:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768013.675846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768013.675854:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768013.675857:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768013.675861:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3742 00000100:00080000:1.0:1775768013.675865:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768013.675869:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768013.675874:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768013.675887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861521280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768013.675929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768013.675980:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffffa0e9c8def800 x1862023861521024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768013.675996:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861521024/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768013.676004:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768013.676006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768013.676009:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861521024:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768013.676029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768013.676050:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768013.676058:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffffa0e9c8def480 x1862023861521152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768013.676066:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861521152/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768013.676070:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768013.676072:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768013.676074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521152:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768013.676077:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768013.676090:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768013.676096:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffffa0e9c8dedc00 x1862023861521280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768013.676102:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861521280/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768013.676115:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768013.676117:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768013.676119:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521280:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768014.251665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768014.507605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768015.275656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768015.531622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768016.299652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768016.555608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768017.323675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768017.579619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768018.347675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768018.603587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768018.795584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768018.795595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768018.795601:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768018.795614:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3747 00000100:00080000:2.0:1775768018.795628:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768018.795636:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768018.795682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861521408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768018.795713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768018.795740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768018.795743:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768018.795749:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3747 00000100:00080000:2.0:1775768018.795755:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768018.795760:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768018.795766:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861521408:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775768018.795774:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768018.795792:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861521536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768018.795806:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768018.795811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768018.795812:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768018.795817:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3747 00000100:00080000:2.0:1775768018.795821:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768018.795826:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768018.795832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768018.795839:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775768018.795846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861521664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768018.795903:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffffa0e9d0712d80 x1862023861521408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768018.795922:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861521408/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768018.795930:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768018.795933:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768018.795937:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861521408:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768018.795962:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768018.795981:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768018.795990:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffffa0e9d0710700 x1862023861521536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768018.795999:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861521536/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768018.796004:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768018.796013:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768018.796016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521536:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768018.796020:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768018.796041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768018.796049:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffffa0e9d0711f80 x1862023861521664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768018.796057:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861521664/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768018.796062:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768018.796064:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768018.796066:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521664:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768019.371689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768019.627569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768020.395647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768020.651571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768021.419719:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768021.675616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768022.443630:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768022.699618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768023.467688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768023.723624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768023.915605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768023.915617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768023.915623:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768023.915635:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3752 00000100:00080000:2.0:1775768023.915649:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768023.915656:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768023.915701:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861521792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768023.915728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768023.915731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768023.915732:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768023.915735:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3752 00000100:00080000:2.0:1775768023.915738:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768023.915741:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768023.915749:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768023.915758:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861521920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768023.915766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768023.915769:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768023.915770:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768023.915774:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3752 00000100:00080000:2.0:1775768023.915776:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768023.915779:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768023.915782:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768023.915803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861521792:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768023.915808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861522048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768023.915927:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521920:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768023.915967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522048:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768024.491706:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768024.747587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768025.515716:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768025.771614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768026.539658:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768026.795578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775768026.987764:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775768026.987807:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768026.987829:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768026.987887:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768026.987898:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775768026.988022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768026] req@ffffa0e9d0710000 x1862023861522048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768026.988055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861522048/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768026.988105:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768026.988112:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768026.988118:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768026.988146:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768026] req@ffffa0e9d0711c00 x1862023861521792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768026.988161:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861521792/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768026.988170:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768026.988173:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768026.988177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861521792:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768026.988184:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768026] req@ffffa0e9d0713b80 x1862023861521920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768026.988194:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861521920/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768026.988201:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768026.988205:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768026.988208:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861521920:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768027.563700:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768027.819590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768028.587643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768028.843615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768029.035603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768029.035613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768029.035617:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768029.035626:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3757 00000100:00080000:0.0:1775768029.035652:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768029.035658:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768029.035689:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861522176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768029.035718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768029.035721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768029.035722:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768029.035725:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3757 00000100:00080000:0.0:1775768029.035731:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768029.035734:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768029.035742:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768029.035753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861522304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768029.035765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768029.035766:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861522176:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768029.035772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768029.035775:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768029.035780:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3757 00000100:00080000:0.0:1775768029.035783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768029.035787:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768029.035792:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768029.035805:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861522432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768029.035838:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768029.035884:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768029/real 1775768029] req@ffffa0e9d09ddc00 x1862023861522176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768029.035919:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861522176/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768029.035930:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768029.035935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768029.035939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861522176:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768029.035965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768029.035992:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768029.036006:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768029/real 1775768029] req@ffffa0e9d09dd180 x1862023861522304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768029.036018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861522304/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768029.036026:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768029.036030:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768029.036033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522304:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768029.036040:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768029.036062:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768029.036073:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768029/real 1775768029] req@ffffa0e9d09dd500 x1862023861522432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768029.036084:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861522432/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768029.036119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768029.036124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768029.036127:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522432:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768029.611679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768029.867615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768030.635690:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768030.891656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768031.659677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768031.915618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768032.683673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768032.939588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768033.707660:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768033.963620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768034.155627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768034.155636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768034.155641:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768034.155654:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3762 00000100:00080000:2.0:1775768034.155666:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768034.155692:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768034.155728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861522560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768034.155756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768034.155759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768034.155760:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768034.155763:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3762 00000100:00080000:2.0:1775768034.155766:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768034.155768:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768034.155777:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768034.155786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861522688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768034.155794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768034.155798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768034.155799:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768034.155802:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3762 00000100:00080000:2.0:1775768034.155805:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768034.155807:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768034.155811:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768034.155819:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861522560:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768034.155824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861522816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768034.155902:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768034.155942:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768034/real 1775768034] req@ffffa0e9d0711180 x1862023861522560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768034.155958:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861522560/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768034.155970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768034.155975:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768034.155979:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861522560:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768034.155989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768034.156014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768034.156028:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768034/real 1775768034] req@ffffa0e9d0710380 x1862023861522688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768034.156041:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861522688/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768034.156050:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768034.156070:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768034.156074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522688:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768034.156110:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768034.156127:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768034.156135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768034/real 1775768034] req@ffffa0e9d0713b80 x1862023861522816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768034.156144:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861522816/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768034.156149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768034.156152:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768034.156154:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861522816:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768034.731725:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768034.987634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768035.755659:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768036.011624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768036.779709:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768037.035602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768037.803664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768038.059580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768038.827689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768039.083577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768039.275635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768039.275645:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768039.275657:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768039.275667:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3767 00000100:00080000:3.0:1775768039.275680:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768039.275687:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768039.275728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861522944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768039.275755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768039.275758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768039.275760:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768039.275763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3767 00000100:00080000:3.0:1775768039.275765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768039.275784:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768039.275794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768039.275804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861523072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768039.275813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768039.275816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768039.275818:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768039.275824:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3767 00000100:00100000:1.0:1775768039.275828:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861522944:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775768039.275829:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768039.275835:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768039.275842:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768039.275850:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861523200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768039.275909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768039.275950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffffa0e9d0541c00 x1862023861522944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768039.275971:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861522944/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768039.275982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768039.275988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768039.275991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861522944:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768039.276029:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768039.276054:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768039.276066:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffffa0e9d0540380 x1862023861523072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768039.276090:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861523072/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768039.276099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768039.276103:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768039.276107:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523072:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768039.276114:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768039.276136:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768039.276149:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffffa0e9d0542300 x1862023861523200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768039.276161:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861523200/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768039.276170:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768039.276174:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768039.276177:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523200:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768039.851627:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768040.107578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768040.875674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768041.131753:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768041.899627:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768042.155582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768042.923669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768043.179622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768043.947712:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768044.203577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768044.395639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768044.395651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768044.395657:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768044.395668:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3772 00000100:00080000:3.0:1775768044.395680:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768044.395688:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768044.395728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861523328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768044.395761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768044.395765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768044.395785:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768044.395790:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3772 00000100:00080000:3.0:1775768044.395796:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768044.395800:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768044.395812:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768044.395830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861523456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768044.395832:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861523328:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775768044.395849:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768044.395854:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768044.395855:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768044.395860:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3772 00000100:00080000:3.0:1775768044.395864:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768044.395868:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768044.395883:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768044.395897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861523584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768044.395909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768044.395985:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffffa0e9d0543800 x1862023861523328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.396013:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861523328/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768044.396036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.396041:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768044.396046:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861523328:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768044.396072:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768044.396098:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768044.396111:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffffa0e9d0540700 x1862023861523456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.396124:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861523456/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768044.396132:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.396136:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768044.396140:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523456:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768044.396146:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768044.396167:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768044.396177:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffffa0e9d0540e00 x1862023861523584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.396189:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861523584/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768044.396197:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.396200:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768044.396204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523584:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768044.971690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768045.227609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768045.995693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768046.251571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768047.019647:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768047.275596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768048.043654:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768048.299559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768049.067675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768049.323598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768049.515599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768049.515608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768049.515614:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768049.515623:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3778 00000100:00080000:3.0:1775768049.515635:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768049.515641:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768049.515680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861523712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768049.515727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768049.515729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768049.515731:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768049.515733:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3778 00000100:00080000:3.0:1775768049.515736:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768049.515739:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768049.515748:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768049.515758:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861523840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768049.515764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768049.515767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768049.515769:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768049.515771:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3778 00000100:00080000:3.0:1775768049.515773:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768049.515775:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768049.515779:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768049.515781:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861523712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775768049.515787:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861523968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768049.515849:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768049.515885:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffffa0e9d0540a80 x1862023861523712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768049.515903:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861523712/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768049.515928:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768049.515932:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768049.515936:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861523712:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768049.515944:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523840:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768049.515965:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768049.515974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffffa0e9d0542d80 x1862023861523840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768049.515982:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861523840/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768049.515987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768049.515990:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768049.515992:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523840:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768049.516013:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768049.516024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768049.516029:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffffa0e9d0541f80 x1862023861523968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768049.516043:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861523968/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768049.516047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768049.516049:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768049.516051:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861523968:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768050.091666:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768050.347606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768051.115697:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768051.371624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768052.139636:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768052.395639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768053.163656:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768053.419637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768054.187638:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768054.443571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768054.635702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768054.635722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768054.635735:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768054.635758:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3783 00000100:00080000:1.0:1775768054.635778:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768054.635786:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768054.635849:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861524096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768054.635940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768054.635944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768054.635946:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768054.635951:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3783 00000100:00080000:1.0:1775768054.635956:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768054.635972:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768054.635973:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861524096:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775768054.635998:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768054.636014:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861524224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768054.636031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768054.636078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffffa0e9c8dee680 x1862023861524096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768054.636085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768054.636091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768054.636093:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768054.636099:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3783 00000100:00100000:0.0:1775768054.636103:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861524096/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768054.636105:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768054.636113:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00080000:0.0:1775768054.636117:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775768054.636121:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768054.636122:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768054.636128:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861524096:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768054.636147:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768054.636150:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861524352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768054.636170:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768054.636183:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffffa0e9c8ded880 x1862023861524224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768054.636196:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861524224/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768054.636206:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768054.636210:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768054.636221:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524224:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768054.636233:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768054.636255:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768054.636265:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffffa0e9c8deea00 x1862023861524352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768054.636276:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861524352/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768054.636281:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768054.636282:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768054.636284:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524352:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768055.211667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768055.467602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768056.235651:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768056.491570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768057.259628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768057.515594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768058.283665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768058.539549:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768059.307624:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768059.563659:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768059.755578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768059.755588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768059.755594:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768059.755604:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3788 00000100:00080000:0.0:1775768059.755617:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768059.755624:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768059.755670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861524480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768059.755702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768059.755705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768059.755707:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768059.755712:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3788 00000100:00100000:1.0:1775768059.755731:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861524480:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768059.755741:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768059.755747:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768059.755758:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768059.755772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861524608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768059.755782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768059.755785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768059.755787:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768059.755791:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3788 00000100:00080000:0.0:1775768059.755795:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775768059.755799:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775768059.755801:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768059.755818:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768059.755831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861524736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768059.755859:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffffa0e9d09df800 x1862023861524480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768059.755874:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861524480/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768059.755881:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768059.755884:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768059.755887:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861524480:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768059.755906:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524608:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768059.755922:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768059.755929:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffffa0e9d09dc700 x1862023861524608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768059.755943:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861524608/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768059.755948:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768059.755951:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768059.755952:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524608:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768059.755956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768059.755971:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768059.755980:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffffa0e9d09dd880 x1862023861524736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768059.755992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861524736/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768059.755999:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768059.756002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768059.756006:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524736:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768060.331608:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768060.587607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768061.355654:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768061.611580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768062.379685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768062.635596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768063.403663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768063.659568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768064.427649:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768064.683621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768064.875635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768064.875646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768064.875650:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768064.875667:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3793 00000100:00080000:2.0:1775768064.875676:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768064.875682:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768064.875721:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861524864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768064.875748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768064.875751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768064.875752:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768064.875764:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3793 00000100:00080000:2.0:1775768064.875767:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768064.875787:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768064.875796:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768064.875801:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861524864:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768064.875810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861524992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768064.875822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768064.875825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768064.875827:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768064.875831:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3793 00000100:00080000:2.0:1775768064.875836:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768064.875839:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768064.875844:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775768064.875855:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:2.0:1775768064.875858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861525120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768064.875894:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffffa0e9d0711c00 x1862023861524864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768064.875910:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861524864/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768064.875931:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768064.875935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768064.875939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861524864:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768064.875971:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524992:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768064.875989:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768064.875997:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffffa0e9d0710000 x1862023861524992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768064.876006:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861524992/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768064.876013:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768064.876016:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768064.876019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861524992:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768064.876025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768064.876043:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768064.876053:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffffa0e9d0711f80 x1862023861525120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768064.876064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861525120/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768064.876071:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768064.876074:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768064.876077:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525120:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768065.451616:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768065.707562:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768066.475647:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768066.731599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768067.499625:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768067.755566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768068.523676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768068.779595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768069.547649:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768069.803620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768069.995592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768069.995601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768069.995605:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768069.995613:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3798 00000100:00080000:0.0:1775768069.995622:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768069.995628:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768069.995663:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861525248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768069.995702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768069.995705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768069.995717:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768069.995721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3798 00000100:00080000:0.0:1775768069.995724:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768069.995727:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768069.995735:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768069.995745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861525376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768069.995753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768069.995756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768069.995757:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768069.995760:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3798 00000100:00080000:0.0:1775768069.995763:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768069.995765:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768069.995770:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768069.995770:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861525248:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768069.995780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861525504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768069.995832:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768069.995869:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffffa0e9d09ded80 x1862023861525248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768069.995882:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861525248/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768069.995890:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768069.995894:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768069.995897:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861525248:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768069.995902:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525376:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768069.995915:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768069.995922:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffffa0e9d09dfb80 x1862023861525376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768069.995930:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861525376/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768069.995935:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768069.995937:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768069.995939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525376:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768069.995956:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768069.995967:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768069.995972:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffffa0e9d09df100 x1862023861525504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768069.995988:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861525504/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768069.995993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768069.995995:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768069.995997:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525504:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768070.571739:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768070.827545:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768071.595620:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768071.851601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768072.619698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768072.875616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768073.643705:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768073.899569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768074.667676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768074.923603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768075.115602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768075.115615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768075.115630:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768075.115645:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3803 00000100:00080000:1.0:1775768075.115668:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768075.115677:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768075.115721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861525632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768075.115751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768075.115754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768075.115756:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768075.115759:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3803 00000100:00080000:1.0:1775768075.115762:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768075.115765:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768075.115774:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768075.115784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861525760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768075.115803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861525632:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768075.115819:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768075.115821:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768075.115823:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768075.115826:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3803 00000100:00080000:1.0:1775768075.115830:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768075.115833:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768075.115836:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768075.115845:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861525888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768075.115880:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768075.115921:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768075/real 1775768075] req@ffffa0e9c8deca80 x1862023861525632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768075.115944:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861525632/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768075.115952:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768075.115956:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768075.115959:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861525632:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768075.115977:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525760:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768075.116006:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768075.116015:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768075/real 1775768075] req@ffffa0e9c8dedf80 x1862023861525760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768075.116024:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861525760/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768075.116029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768075.116031:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768075.116033:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525760:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768075.116037:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525888:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768075.116048:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768075.116054:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768075/real 1775768075] req@ffffa0e9c8dece00 x1862023861525888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768075.116061:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861525888/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768075.116075:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768075.116078:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768075.116080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861525888:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768075.691712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768075.947577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768076.715691:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768076.971596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768077.739717:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768077.995612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768078.763712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768079.019616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768079.787722:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768080.043625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768080.235639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768080.235649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768080.235653:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768080.235662:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3808 00000100:00080000:1.0:1775768080.235672:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768080.235678:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768080.235715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861526016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768080.235744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768080.235749:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768080.235751:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768080.235755:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3808 00000100:00080000:1.0:1775768080.235759:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768080.235764:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768080.235791:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768080.235798:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861526016:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768080.235804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861526144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768080.235814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768080.235816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768080.235818:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768080.235820:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3808 00000100:00080000:1.0:1775768080.235824:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768080.235827:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768080.235831:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768080.235838:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861526272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768080.235858:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768080.235895:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffffa0e9c8dec700 x1862023861526016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768080.235908:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861526016/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768080.235916:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768080.235919:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768080.235922:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861526016:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768080.235938:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526144:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768080.235953:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768080.235961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffffa0e9c8defb80 x1862023861526144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768080.235968:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861526144/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768080.235974:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768080.235983:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768080.235985:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526144:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768080.235989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768080.236003:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768080.236009:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffffa0e9c8def100 x1862023861526272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768080.236017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861526272/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768080.236021:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768080.236023:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768080.236025:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526272:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768080.811643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768081.067564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768081.835640:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768082.091632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768082.859699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768083.115578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768083.883718:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768084.139640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768084.907669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768085.163607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768085.355608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768085.355618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768085.355622:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768085.355631:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3813 00000100:00080000:0.0:1775768085.355640:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768085.355646:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768085.355683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861526400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768085.355708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768085.355711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768085.355731:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768085.355734:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3813 00000100:00080000:0.0:1775768085.355738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775768085.355740:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861526400:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768085.355744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768085.355754:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768085.355768:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861526528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768085.355780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768085.355784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768085.355786:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768085.355790:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3813 00000100:00080000:0.0:1775768085.355795:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775768085.355800:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775768085.355801:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768085.355809:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768085.355823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861526656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768085.355852:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffffa0e9d09dce00 x1862023861526400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768085.355873:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861526400/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768085.355884:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768085.355889:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768085.355893:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861526400:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768085.355916:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526528:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768085.355950:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768085.355964:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffffa0e9d09dc380 x1862023861526528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768085.355978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861526528/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768085.355985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768085.355988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768085.355990:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526528:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768085.355995:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768085.356014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768085.356025:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffffa0e9d09dc000 x1862023861526656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768085.356038:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861526656/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768085.356047:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768085.356051:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768085.356054:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526656:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768085.931702:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768086.187635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768086.955655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768087.211619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768087.979660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768088.235624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768089.003702:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768089.259608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768090.027673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768090.283585:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768090.475630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768090.475642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768090.475647:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768090.475658:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3818 00000100:00080000:2.0:1775768090.475694:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768090.475701:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768090.475741:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861526784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768090.475779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768090.475783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768090.475784:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768090.475788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3818 00000100:00080000:2.0:1775768090.475791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768090.475796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768090.475807:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768090.475823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861526912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768090.475837:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861526784:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768090.475837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768090.475846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768090.475849:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768090.475854:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3818 00000100:00080000:2.0:1775768090.475858:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768090.475863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768090.475869:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768090.475884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861527040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768090.475946:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526912:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768090.475970:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527040:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768091.051674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768091.307615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768092.075640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768092.331595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768093.099635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768093.355567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775768093.547839:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775768093.547898:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775768093.547917:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775768093.547975:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775768093.548019:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775768093.548342:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768093] req@ffffa0e9d0712d80 x1862023861526912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768093.548371:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861526912/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768093.548385:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768093.548391:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768093.548395:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861526912:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768093.548405:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768093] req@ffffa0e9d0710a80 x1862023861527040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768093.548424:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861527040/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768093.548433:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768093.548438:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768093.548442:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768093.548450:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768093] req@ffffa0e9d0710700 x1862023861526784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768093.548463:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861526784/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768093.548471:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768093.548474:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768093.548520:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861526784:192.168.202.104@tcp:250:kworker.0 00010000:00010000:0.0:1775768094.123661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768094.379612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768095.147706:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768095.403588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768095.595608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768095.595618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768095.595626:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768095.595635:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3824 00000100:00080000:2.0:1775768095.595646:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768095.595652:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768095.595688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861527168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768095.595719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768095.595724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768095.595726:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768095.595729:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3824 00000100:00080000:2.0:1775768095.595733:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768095.595736:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768095.595746:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768095.595762:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861527296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768095.595774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768095.595776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775768095.595803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861527168:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768095.595805:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768095.595827:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3824 00000100:00080000:2.0:1775768095.595832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768095.595839:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768095.595848:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768095.595866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861527424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768095.595901:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768095.595961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffffa0e9d0710e00 x1862023861527168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768095.595985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861527168/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768095.595997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768095.596002:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768095.596006:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861527168:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768095.596025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527296:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768095.596053:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768095.596068:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffffa0e9d0712680 x1862023861527296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768095.596081:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861527296/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768095.596089:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768095.596093:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768095.596097:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527296:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768095.596132:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768095.596156:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768095.596168:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffffa0e9d0711500 x1862023861527424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768095.596197:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861527424/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768095.596206:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768095.596210:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768095.596213:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527424:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768096.171633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768096.427635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768097.195702:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768097.451619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768098.219682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768098.475653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768099.243733:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768099.499623:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768100.267642:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768100.523607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768100.715639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768100.715647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768100.715651:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768100.715660:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3829 00000100:00080000:0.0:1775768100.715694:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768100.715702:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768100.715744:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861527552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768100.715768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768100.715771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768100.715773:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768100.715776:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3829 00000100:00080000:0.0:1775768100.715779:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768100.715781:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768100.715790:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768100.715798:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861527680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768100.715806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768100.715808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768100.715810:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768100.715812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3829 00000100:00080000:0.0:1775768100.715815:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768100.715817:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768100.715821:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768100.715825:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861527552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768100.715830:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861527808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768100.715909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768100.715965:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768100] req@ffffa0e9d09de300 x1862023861527552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768100.715985:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861527552/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768100.715996:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768100.716001:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768100.716005:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861527552:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768100.716013:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527680:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768100.716038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768100.716052:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768100] req@ffffa0e9d09dea00 x1862023861527680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768100.716065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861527680/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768100.716087:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768100.716091:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768100.716095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527680:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768100.716121:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768100.716144:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768100.716157:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768100] req@ffffa0e9d09df480 x1862023861527808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768100.716169:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861527808/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768100.716178:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768100.716181:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768100.716184:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861527808:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768101.291694:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768101.547599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768102.315663:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768102.571593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768103.339659:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768103.595619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768104.363646:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768104.619554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768105.387660:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768105.643635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768105.835621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768105.835630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768105.835634:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768105.835643:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3834 00000100:00080000:0.0:1775768105.835653:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768105.835660:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768105.835698:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861527936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768105.835718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768105.835721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768105.835723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768105.835725:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3834 00000100:00080000:0.0:1775768105.835728:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768105.835731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768105.835739:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768105.835749:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861527936:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768105.835750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861528064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768105.835763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768105.835766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768105.835767:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768105.835769:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3834 00000100:00080000:0.0:1775768105.835772:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768105.835774:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768105.835777:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768105.835801:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861528192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768105.835815:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768105.835850:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffffa0e9d09df480 x1862023861527936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768105.835864:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861527936/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768105.835872:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768105.835875:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768105.835885:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861527936:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768105.835904:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528064:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768105.835920:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768105.835929:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffffa0e9d09dea00 x1862023861528064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768105.835937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861528064/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768105.835942:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768105.835944:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768105.835946:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528064:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768105.835950:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768105.835959:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768105.835965:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffffa0e9d09de300 x1862023861528192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768105.835972:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861528192/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768105.835977:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768105.835979:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768105.835980:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528192:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768106.411683:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768106.667643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768107.435714:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768107.691610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768108.459625:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768108.715632:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768109.483692:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768109.739611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768110.507644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768110.763609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768110.955597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768110.955609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768110.955614:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768110.955625:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3839 00000100:00080000:2.0:1775768110.955637:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768110.955645:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768110.955685:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861528320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768110.955718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768110.955722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768110.955723:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768110.955726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3839 00000100:00080000:2.0:1775768110.955730:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768110.955733:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768110.955741:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768110.955753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861528448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768110.955767:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861528320:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768110.955768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768110.955776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768110.955777:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768110.955781:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3839 00000100:00080000:2.0:1775768110.955783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768110.955808:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768110.955813:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768110.955824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861528576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768110.955836:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768110.955882:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffffa0e9d0713480 x1862023861528320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768110.955896:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861528320/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768110.955904:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768110.955907:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768110.955911:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861528320:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768110.955931:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528448:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768110.955947:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768110.955956:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffffa0e9d0711880 x1862023861528448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768110.955969:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861528448/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768110.955978:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768110.955981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768110.955987:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528448:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768110.955992:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768110.956013:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768110.956024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffffa0e9d0712a00 x1862023861528576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768110.956035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861528576/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768110.956055:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768110.956059:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768110.956062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528576:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768111.531699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768111.787591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768112.555705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768112.811574:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768113.579687:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768113.835617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768114.603667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768114.859579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768115.627638:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768115.883636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768116.075649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768116.075661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768116.075667:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768116.075679:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3844 00000100:00080000:1.0:1775768116.075694:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768116.075701:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768116.075747:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861528704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768116.075787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768116.075791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768116.075793:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768116.075797:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3844 00000100:00080000:1.0:1775768116.075802:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768116.075806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768116.075817:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768116.075829:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861528832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768116.075841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768116.075845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775768116.075859:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861528704:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768116.075862:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768116.075869:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3844 00000100:00080000:1.0:1775768116.075874:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768116.075878:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768116.075884:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768116.075894:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861528960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768116.075929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768116.075975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768116/real 1775768116] req@ffffa0e9c8def100 x1862023861528704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.075992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861528704/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.076002:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.076005:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768116.076008:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861528704:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768116.076014:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528832:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768116.076030:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768116.076039:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768116/real 1775768116] req@ffffa0e9c8defb80 x1862023861528832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.076047:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861528832/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.076052:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.076054:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768116.076062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528832:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768116.076085:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768116.076099:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768116.076106:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768116/real 1775768116] req@ffffa0e9c8dec700 x1862023861528960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.076114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861528960/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.076119:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.076121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768116.076124:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861528960:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768116.651759:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768116.907567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768117.675594:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768117.931595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768118.699687:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768118.955635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768119.723614:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768119.979584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768120.747668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768121.003626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768121.195610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768121.195618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768121.195626:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768121.195636:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3849 00000100:00080000:0.0:1775768121.195645:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768121.195650:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768121.195683:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861529088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768121.195704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768121.195707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768121.195708:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768121.195711:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3849 00000100:00080000:0.0:1775768121.195729:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768121.195733:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768121.195733:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861529088:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775768121.195744:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768121.195755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861529216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768121.195762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768121.195765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768121.195766:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768121.195768:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3849 00000100:00080000:0.0:1775768121.195784:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768121.195786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768121.195788:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775768121.195793:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768121.195796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861529344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768121.195829:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffffa0e9d09dc000 x1862023861529088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768121.195841:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861529088/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768121.195848:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768121.195851:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768121.195854:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861529088:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768121.195870:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529216:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768121.195883:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768121.195891:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffffa0e9d09dc380 x1862023861529216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768121.195902:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861529216/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768121.195921:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768121.195925:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768121.195928:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529216:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768121.195934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768121.195951:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768121.195959:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffffa0e9d09dce00 x1862023861529344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768121.195965:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861529344/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768121.195970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768121.195972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768121.195974:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529344:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768121.771695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768122.027577:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768122.795656:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768123.051602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768123.819654:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768124.075565:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768124.843609:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768125.099570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768125.867652:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768126.123617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768126.315618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768126.315628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768126.315632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768126.315641:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3854 00000100:00080000:0.0:1775768126.315650:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768126.315656:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768126.315688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861529472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768126.315730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768126.315732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768126.315734:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768126.315736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3854 00000100:00080000:0.0:1775768126.315739:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768126.315741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768126.315749:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768126.315763:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861529600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768126.315771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768126.315772:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861529472:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768126.315774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768126.315777:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768126.315780:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3854 00000100:00080000:0.0:1775768126.315782:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768126.315784:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768126.315787:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768126.315795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861529728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768126.315828:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768126.315869:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffffa0e9d09df100 x1862023861529472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768126.315883:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861529472/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768126.315889:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768126.315892:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768126.315900:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861529472:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768126.315917:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529600:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768126.315934:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768126.315949:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffffa0e9d09dfb80 x1862023861529600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768126.315961:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861529600/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768126.315969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768126.315972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768126.315975:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529600:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768126.315980:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768126.316008:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768126.316019:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffffa0e9d09ded80 x1862023861529728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768126.316027:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861529728/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768126.316032:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768126.316034:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768126.316036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529728:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768126.891759:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768127.147611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768127.915633:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768128.171603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768128.939602:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768129.195597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768129.963633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768130.219587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768130.987668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768131.243592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768131.435588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768131.435596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768131.435600:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768131.435608:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3859 00000100:00080000:3.0:1775768131.435617:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768131.435623:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768131.435656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861529856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768131.435677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768131.435679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768131.435680:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768131.435683:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3859 00000100:00080000:3.0:1775768131.435685:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768131.435686:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768131.435694:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768131.435707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861529984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768131.435715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768131.435717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775768131.435718:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861529856:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775768131.435718:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768131.435725:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3859 00000100:00080000:3.0:1775768131.435727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768131.435745:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768131.435749:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768131.435758:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861530112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768131.435783:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768131.435832:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffffa0e9d0541180 x1862023861529856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768131.435846:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861529856/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768131.435854:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768131.435856:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768131.435859:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861529856:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768131.435876:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529984:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768131.435892:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768131.435900:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffffa0e9d0543480 x1862023861529984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768131.435912:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861529984/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768131.435917:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768131.435919:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768131.435921:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861529984:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768131.435925:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768131.435941:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768131.435950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffffa0e9d0543100 x1862023861530112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768131.435962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861530112/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768131.435981:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768131.435984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768131.435987:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530112:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768132.011620:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768132.267579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768133.035631:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768133.291566:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768134.059657:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768134.315579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768135.083650:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768135.339592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768136.107667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768136.363554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768136.555612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768136.555620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768136.555624:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768136.555631:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3865 00000100:00080000:3.0:1775768136.555640:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768136.555644:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768136.555678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861530240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768136.555699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768136.555701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768136.555702:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768136.555704:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3865 00000100:00080000:3.0:1775768136.555706:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768136.555708:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768136.555715:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768136.555723:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861530368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768136.555729:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768136.555732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775768136.555741:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861530240:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775768136.555748:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768136.555751:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3865 00000100:00080000:3.0:1775768136.555754:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768136.555757:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768136.555760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768136.555769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861530496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768136.555798:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768136.555840:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffffa0e9d0543100 x1862023861530240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768136.555857:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861530240/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768136.555864:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768136.555867:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768136.555869:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861530240:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768136.555874:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530368:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768136.555889:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768136.555897:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffffa0e9d0543480 x1862023861530368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768136.555903:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861530368/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768136.555908:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768136.555910:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768136.555912:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530368:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768136.555934:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768136.555947:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768136.555953:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffffa0e9d0541180 x1862023861530496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768136.555960:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861530496/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768136.555964:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768136.555966:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768136.555967:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530496:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768137.131637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768137.387612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768138.155676:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768138.411621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768139.179684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768139.435618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768140.203637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768140.459638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768141.227669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768141.483617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768141.675615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768141.675622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768141.675626:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768141.675635:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3870 00000100:00080000:1.0:1775768141.675646:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768141.675652:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768141.675687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861530624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768141.675723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768141.675727:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768141.675729:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768141.675733:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3870 00000100:00080000:1.0:1775768141.675737:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768141.675754:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768141.675763:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768141.675779:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861530752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768141.675792:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768141.675795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768141.675796:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768141.675800:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3870 00000100:00080000:1.0:1775768141.675806:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768141.675807:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861530624:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768141.675813:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768141.675824:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768141.675841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861530880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768141.675884:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768141.675937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffffa0e9c8dece00 x1862023861530624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768141.675956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861530624/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768141.675966:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768141.675971:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768141.675975:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861530624:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768141.675997:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530752:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768141.676027:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768141.676036:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffffa0e9c8dedf80 x1862023861530752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768141.676044:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861530752/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768141.676049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768141.676052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768141.676053:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530752:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768141.676057:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530880:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768141.676069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768141.676074:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffffa0e9c8deca80 x1862023861530880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768141.676094:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861530880/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768141.676099:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768141.676101:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768141.676102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861530880:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768142.251643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768142.507590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768143.275602:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768143.531541:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768144.299644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768144.555605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768145.323633:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768145.579570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768146.347683:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768146.603605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768146.795597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768146.795604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768146.795609:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768146.795621:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3875 00000100:00080000:1.0:1775768146.795633:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768146.795641:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768146.795692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861531008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768146.795720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768146.795724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768146.795737:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768146.795742:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3875 00000100:00080000:1.0:1775768146.795746:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768146.795751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768146.795761:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768146.795765:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861531008:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768146.795775:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861531136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768146.795786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768146.795790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768146.795791:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768146.795796:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3875 00000100:00080000:1.0:1775768146.795799:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768146.795802:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768146.795806:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768146.795814:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861531264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768146.795829:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768146.795871:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffffa0e9c8deea00 x1862023861531008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768146.795888:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861531008/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768146.795896:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768146.795898:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768146.795901:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861531008:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768146.795919:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531136:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768146.795932:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768146.795942:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffffa0e9c8ded880 x1862023861531136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768146.795953:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861531136/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768146.795977:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768146.795981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768146.795985:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531136:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768146.795991:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768146.796012:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768146.796023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffffa0e9c8dee680 x1862023861531264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768146.796034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861531264/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768146.796041:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768146.796043:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768146.796045:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531264:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768147.371661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768147.627615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768148.395702:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768148.651615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768149.419628:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768149.675614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768150.443624:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768150.699584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768151.467594:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768151.723600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768151.915573:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768151.915582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768151.915588:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768151.915603:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3880 00000100:00080000:1.0:1775768151.915616:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768151.915624:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768151.915665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861531392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768151.915711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768151.915714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768151.915716:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768151.915720:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3880 00000100:00080000:1.0:1775768151.915724:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768151.915727:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768151.915737:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768151.915750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861531520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768151.915759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768151.915761:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861531392:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768151.915767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768151.915770:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768151.915774:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3880 00000100:00080000:1.0:1775768151.915776:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768151.915780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768151.915785:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768151.915795:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861531648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768151.915837:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768151.915890:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffffa0e9c8dedc00 x1862023861531392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768151.915912:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861531392/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768151.915945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768151.915948:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768151.915956:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861531392:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768151.915976:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531520:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768151.915994:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768151.916002:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffffa0e9c8def480 x1862023861531520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768151.916010:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861531520/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768151.916015:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768151.916017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768151.916019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531520:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768151.916022:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768151.916031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768151.916037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffffa0e9c8def800 x1862023861531648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768151.916043:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861531648/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768151.916048:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768151.916049:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768151.916051:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531648:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768152.491678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768152.747610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768153.515670:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768153.771553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768154.539646:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768154.795584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768155.563698:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768155.819641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768156.587628:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768156.843604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768157.035580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768157.035588:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768157.035591:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768157.035599:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3885 00000100:00080000:1.0:1775768157.035626:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768157.035631:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768157.035657:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861531776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768157.035679:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768157.035682:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768157.035683:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768157.035686:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3885 00000100:00080000:1.0:1775768157.035688:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768157.035690:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768157.035697:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768157.035705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861531904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768157.035713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768157.035715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768157.035716:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768157.035720:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3885 00000100:00100000:0.0:1775768157.035721:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861531776:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768157.035723:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768157.035728:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768157.035731:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768157.035738:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861532032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768157.035805:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531904:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768157.035825:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532032:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768157.611677:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768157.867619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768158.635620:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768158.891584:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768159.659658:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768159.915603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775768160.107809:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775768160.107882:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768160.107900:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768160.107948:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768160.107959:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775768160.108011:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768157/real 1775768160] req@ffffa0e9c8deed80 x1862023861531904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768160.108045:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861531904/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768160.108062:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768160.108081:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768160.108087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861531904:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768160.108095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768157/real 1775768160] req@ffffa0e9c8dec380 x1862023861532032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768160.108108:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861532032/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768160.108115:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768160.108119:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768160.108122:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768160.108152:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768157/real 1775768160] req@ffffa0e9c8ded500 x1862023861531776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768160.108165:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861531776/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768160.108172:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768160.108176:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768160.108179:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861531776:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775768160.683638:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768160.939552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768161.707669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768161.963611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768162.155623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768162.155650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768162.155655:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768162.155664:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3890 00000100:00080000:1.0:1775768162.155674:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768162.155681:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768162.155715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861532160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768162.155755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768162.155759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768162.155761:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768162.155765:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3890 00000100:00080000:1.0:1775768162.155768:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768162.155772:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768162.155781:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768162.155794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861532288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768162.155802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768162.155805:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768162.155807:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768162.155810:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3890 00000100:00080000:1.0:1775768162.155813:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768162.155832:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768162.155836:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768162.155846:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861532416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768162.155957:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861532160:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775768162.156036:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768162.156095:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768162/real 1775768162] req@ffffa0e9c8dee300 x1862023861532160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768162.156112:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861532160/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768162.156122:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768162.156126:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768162.156130:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861532160:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768162.156138:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532288:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768162.156157:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768162.156167:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768162/real 1775768162] req@ffffa0e9c8dec000 x1862023861532288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768162.156185:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861532288/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768162.156192:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768162.156195:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768162.156198:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532288:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768162.156203:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768162.156220:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768162.156229:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768162/real 1775768162] req@ffffa0e9c8ded180 x1862023861532416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768162.156239:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861532416/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768162.156246:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768162.156249:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768162.156252:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532416:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768162.731634:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768162.987575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768163.755661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768164.011641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768164.779720:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768165.035613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768165.803767:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768166.059599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768166.827664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768167.083615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768167.275568:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768167.275576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768167.275594:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768167.275603:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3895 00000100:00080000:0.0:1775768167.275613:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768167.275619:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768167.275653:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861532544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768167.275676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768167.275678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768167.275679:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768167.275682:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3895 00000100:00080000:0.0:1775768167.275684:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768167.275686:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768167.275695:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768167.275702:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861532672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768167.275710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768167.275714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775768167.275715:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861532544:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768167.275716:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768167.275723:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3895 00000100:00080000:0.0:1775768167.275725:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768167.275727:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768167.275730:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768167.275738:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861532800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768167.275792:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768167.275829:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768167] req@ffffa0e9d09dd880 x1862023861532544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768167.275843:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861532544/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768167.275851:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768167.275854:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768167.275857:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861532544:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768167.275875:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532672:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768167.275889:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768167.275896:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768167] req@ffffa0e9d09dc700 x1862023861532672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768167.275903:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861532672/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768167.275909:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768167.275911:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768167.275913:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532672:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768167.275915:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768167.275924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768167.275936:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768167] req@ffffa0e9d09df800 x1862023861532800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768167.275944:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861532800/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768167.275949:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768167.275950:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768167.275952:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861532800:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768167.851663:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768168.107611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768168.875648:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768169.131589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768169.899662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768170.155618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768170.923653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768171.179637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768171.947685:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768172.203580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768172.395565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768172.395572:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768172.395576:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768172.395591:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3900 00000100:00080000:0.0:1775768172.395599:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768172.395604:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768172.395636:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861532928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768172.395665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768172.395669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768172.395671:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768172.395682:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3900 00000100:00080000:0.0:1775768172.395685:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768172.395690:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768172.395698:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768172.395710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861533056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768172.395724:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861532928:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768172.395733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768172.395749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768172.395751:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768172.395754:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3900 00000100:00080000:0.0:1775768172.395757:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768172.395760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768172.395765:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768172.395778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861533184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768172.395787:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768172.395826:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffffa0e9d09dd500 x1862023861532928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768172.395840:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861532928/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768172.395850:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768172.395854:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768172.395858:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861532928:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768172.395879:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533056:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768172.395899:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768172.395907:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffffa0e9d09dd180 x1862023861533056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768172.395915:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861533056/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768172.395929:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768172.395931:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768172.395933:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533056:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768172.395937:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768172.395949:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768172.395956:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffffa0e9d09ddc00 x1862023861533184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768172.395962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861533184/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768172.395966:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768172.395968:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768172.395970:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533184:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768172.971634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768173.227552:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768173.995644:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768174.251580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768175.019721:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768175.275539:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768176.043635:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768176.299599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768177.067693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768177.323625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768177.515608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768177.515616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768177.515620:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768177.515636:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3906 00000100:00080000:2.0:1775768177.515652:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768177.515659:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768177.515722:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861533312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768177.515750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768177.515753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768177.515755:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768177.515759:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3906 00000100:00080000:2.0:1775768177.515763:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768177.515767:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768177.515786:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768177.515796:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861533312:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768177.515799:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861533440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768177.515809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768177.515811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768177.515813:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768177.515816:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3906 00000100:00080000:2.0:1775768177.515818:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768177.515821:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768177.515824:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768177.515845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861533568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768177.515869:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768177.515904:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffffa0e9d0712300 x1862023861533312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768177.515919:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861533312/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768177.515926:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768177.515929:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768177.515937:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861533312:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768177.515957:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533440:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768177.515974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768177.515982:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffffa0e9d0713100 x1862023861533440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768177.515989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861533440/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768177.515994:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768177.515996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768177.515998:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533440:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768177.516001:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768177.516010:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768177.516018:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffffa0e9d0712a00 x1862023861533568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768177.516028:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861533568/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768177.516036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768177.516039:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768177.516042:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533568:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768178.091616:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768178.347583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768179.115674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768179.371618:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768180.139696:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768180.395576:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768181.163717:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768181.419680:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768182.187739:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768182.443556:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768182.635587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768182.635596:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768182.635602:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768182.635611:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3911 00000100:00080000:2.0:1775768182.635641:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768182.635648:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768182.635686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861533696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768182.635713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768182.635716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768182.635717:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768182.635720:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3911 00000100:00080000:2.0:1775768182.635723:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768182.635725:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768182.635734:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768182.635742:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861533824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768182.635751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768182.635770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768182.635771:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768182.635776:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3911 00000100:00080000:2.0:1775768182.635780:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768182.635786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768182.635792:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768182.635795:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861533696:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768182.635804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861533952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768182.635869:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768182.635922:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffffa0e9d0711880 x1862023861533696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768182.635937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861533696/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768182.635946:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768182.635949:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768182.635952:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861533696:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768182.635958:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533824:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768182.635976:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768182.635984:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffffa0e9d0713480 x1862023861533824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768182.635992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861533824/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768182.635997:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768182.635999:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768182.636001:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533824:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768182.636023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768182.636034:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768182.636051:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffffa0e9d0711500 x1862023861533952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768182.636059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861533952/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768182.636079:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768182.636083:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768182.636087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861533952:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768183.211712:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768183.467635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768184.235718:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768184.491583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768185.259688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768185.515641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768186.283660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768186.539679:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768187.307683:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768187.563622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768187.755623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768187.755631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768187.755635:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768187.755643:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3916 00000100:00080000:1.0:1775768187.755651:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768187.755657:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768187.755691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861534080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768187.755717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768187.755720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768187.755721:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768187.755723:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3916 00000100:00080000:1.0:1775768187.755725:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768187.755728:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768187.755735:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768187.755744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861534208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768187.755750:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768187.755753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775768187.755761:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861534080:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768187.755769:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768187.755778:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3916 00000100:00080000:1.0:1775768187.755782:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768187.755786:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768187.755791:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768187.755803:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861534336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768187.755833:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768187.755867:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffffa0e9c8ded180 x1862023861534080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768187.755880:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861534080/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768187.755887:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768187.755890:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768187.755892:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861534080:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768187.755897:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534208:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768187.755909:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768187.755916:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffffa0e9c8dec000 x1862023861534208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768187.755923:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861534208/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768187.755928:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768187.755930:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768187.755931:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534208:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768187.755948:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768187.755957:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768187.755962:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffffa0e9c8dee300 x1862023861534336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768187.755977:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861534336/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768187.755982:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768187.755984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768187.755986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534336:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768188.331648:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768188.587614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768189.355766:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768189.611606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768190.379615:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768190.635590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768191.403628:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768191.659617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768192.427670:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768192.683594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768192.875597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768192.875605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768192.875609:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768192.875617:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3921 00000100:00080000:1.0:1775768192.875626:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768192.875632:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768192.875670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861534464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768192.875699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768192.875703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768192.875704:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768192.875708:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3921 00000100:00080000:1.0:1775768192.875712:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768192.875716:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768192.875725:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768192.875739:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861534592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768192.875750:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861534464:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768192.875784:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768192.875788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768192.875790:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768192.875794:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3921 00000100:00080000:1.0:1775768192.875802:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768192.875806:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768192.875812:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768192.875813:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768192.875828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861534720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768192.875857:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffffa0e9c8ded500 x1862023861534464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768192.875875:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861534464/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768192.875882:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768192.875885:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768192.875888:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861534464:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768192.875905:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534592:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768192.875929:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768192.875945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffffa0e9c8dec380 x1862023861534592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768192.875953:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861534592/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768192.875958:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768192.875960:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768192.875962:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534592:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768192.875965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768192.875975:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768192.875981:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffffa0e9c8deed80 x1862023861534720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768192.875987:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861534720/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768192.875992:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768192.875993:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768192.875995:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534720:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768193.451658:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768193.707591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768194.475652:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768194.731602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768195.499619:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768195.755588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768196.523688:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768196.779630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768197.547662:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768197.803593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768197.995621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768197.995629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768197.995633:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768197.995641:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3926 00000100:00080000:2.0:1775768197.995649:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768197.995654:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768197.995688:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861534848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768197.995727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768197.995730:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768197.995731:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768197.995734:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3926 00000100:00080000:2.0:1775768197.995738:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768197.995741:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768197.995750:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768197.995764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861534976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768197.995771:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861534848:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768197.995778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768197.995782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768197.995784:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768197.995789:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3926 00000100:00080000:2.0:1775768197.995793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768197.995797:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768197.995802:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768197.995813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861535104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768197.995826:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768197.995860:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffffa0e9d0712680 x1862023861534848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768197.995873:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861534848/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768197.995879:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768197.995882:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768197.995885:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861534848:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768197.995901:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534976:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768197.995914:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768197.995920:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffffa0e9d0710e00 x1862023861534976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768197.995927:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861534976/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768197.995932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768197.995934:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768197.995935:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861534976:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768197.995938:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768197.995946:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768197.995951:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffffa0e9d0710700 x1862023861535104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768197.995967:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861535104/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768197.995972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768197.995974:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768197.995975:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535104:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768198.571673:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768198.827596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768199.595712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768199.851572:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768200.619673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768200.875567:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768201.643677:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768201.899578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768202.667666:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768202.923627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768203.115660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768203.115670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768203.115675:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768203.115684:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3931 00000100:00080000:1.0:1775768203.115696:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768203.115702:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768203.115742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861535232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768203.115769:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768203.115772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768203.115773:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768203.115776:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3931 00000100:00080000:1.0:1775768203.115778:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768203.115781:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768203.115805:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768203.115816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861535360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768203.115819:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861535232:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768203.115833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768203.115835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768203.115837:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768203.115840:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3931 00000100:00080000:1.0:1775768203.115844:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768203.115846:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768203.115850:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768203.115858:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861535488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768203.115900:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768203.115937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768203/real 1775768203] req@ffffa0e9c8def800 x1862023861535232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768203.115954:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861535232/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768203.115962:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768203.115966:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768203.115969:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861535232:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768203.115989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535360:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768203.116014:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768203.116023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768203/real 1775768203] req@ffffa0e9c8def480 x1862023861535360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768203.116031:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861535360/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768203.116044:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768203.116047:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768203.116049:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535360:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768203.116053:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768203.116065:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768203.116072:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768203/real 1775768203] req@ffffa0e9c8dedc00 x1862023861535488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768203.116079:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861535488/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768203.116084:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768203.116086:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768203.116088:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535488:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768203.691627:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768203.947628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768204.715681:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768204.971617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768205.739633:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768205.995610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768206.763680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768207.019628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768207.787641:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768208.043615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768208.235589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768208.235598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768208.235603:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768208.235612:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3936 00000100:00080000:2.0:1775768208.235623:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768208.235630:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768208.235670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861535616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768208.235695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768208.235698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768208.235699:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768208.235703:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3936 00000100:00080000:2.0:1775768208.235706:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768208.235709:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768208.235718:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768208.235728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861535744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768208.235736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768208.235739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768208.235742:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768208.235743:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861535616:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768208.235747:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3936 00000100:00080000:2.0:1775768208.235752:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768208.235754:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768208.235775:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768208.235787:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861535872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768208.235809:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768208.235854:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffffa0e9d0710a80 x1862023861535616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768208.235875:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861535616/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768208.235886:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768208.235889:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768208.235892:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861535616:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768208.235912:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535744:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768208.235930:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768208.235939:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffffa0e9d0712d80 x1862023861535744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768208.235947:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861535744/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768208.235953:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768208.235955:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768208.235957:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535744:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768208.235961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535872:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768208.235971:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768208.235976:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffffa0e9d0711f80 x1862023861535872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768208.235990:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861535872/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768208.235996:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768208.235998:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768208.236000:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861535872:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768208.811631:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768209.067589:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768209.835769:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768210.091609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768210.859723:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768211.115626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768211.883619:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768212.139626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768212.907604:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768213.163627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768213.355577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768213.355585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768213.355588:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768213.355596:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3941 00000100:00080000:3.0:1775768213.355604:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768213.355609:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768213.355647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861536000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768213.355675:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768213.355679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768213.355680:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768213.355684:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3941 00000100:00080000:3.0:1775768213.355687:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768213.355690:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768213.355698:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768213.355708:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861536128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768213.355721:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861536000:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775768213.355734:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768213.355737:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768213.355738:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768213.355741:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3941 00000100:00080000:3.0:1775768213.355744:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768213.355748:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768213.355753:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768213.355770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861536256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768213.355775:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768213.355818:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffffa0e9d0541f80 x1862023861536000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768213.355837:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861536000/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768213.355847:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768213.355850:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768213.355853:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861536000:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768213.355871:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536128:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768213.355887:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768213.355901:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffffa0e9d0542d80 x1862023861536128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768213.355909:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861536128/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768213.355914:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768213.355916:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768213.355918:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536128:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768213.355921:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768213.355933:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768213.355938:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffffa0e9d0540a80 x1862023861536256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768213.355944:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861536256/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768213.355948:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768213.355950:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768213.355951:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536256:192.168.202.104@tcp:38:kworker.0 00010000:00010000:0.0:1775768213.931687:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768214.187631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768214.955612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768215.211634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768215.979661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768216.235656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768217.003665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768217.259620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768218.027706:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768218.283660:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768218.475619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768218.475627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768218.475631:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768218.475639:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3946 00000100:00080000:2.0:1775768218.475648:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768218.475654:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768218.475688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861536384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768218.475714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768218.475716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768218.475717:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768218.475720:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3946 00000100:00080000:2.0:1775768218.475721:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768218.475723:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768218.475731:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768218.475754:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861536512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768218.475758:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861536384:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768218.475763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768218.475765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768218.475767:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768218.475773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3946 00000100:00080000:2.0:1775768218.475776:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768218.475778:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768218.475781:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768218.475796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861536640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768218.475821:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768218.475852:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffffa0e9d0710000 x1862023861536384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768218.475866:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861536384/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768218.475877:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768218.475881:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768218.475885:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861536384:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768218.475906:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536512:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768218.475927:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768218.475937:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffffa0e9d0711c00 x1862023861536512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768218.475949:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861536512/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768218.475957:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768218.475960:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768218.475963:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536512:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768218.475969:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768218.475987:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768218.475998:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffffa0e9d0713b80 x1862023861536640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768218.476021:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861536640/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768218.476029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768218.476032:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768218.476035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536640:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768219.051680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768219.307569:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768220.075847:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768220.331635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768221.099685:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768221.355590:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768222.123681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768222.379587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768223.147705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768223.403559:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768223.595607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768223.595620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768223.595624:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768223.595634:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3952 00000100:00080000:3.0:1775768223.595644:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768223.595650:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768223.595693:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861536768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768223.595718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768223.595721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768223.595722:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768223.595725:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3952 00000100:00080000:3.0:1775768223.595728:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768223.595731:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768223.595757:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768223.595758:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861536768:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775768223.595772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861536896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768223.595779:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768223.595782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768223.595783:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768223.595786:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3952 00000100:00080000:3.0:1775768223.595789:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768223.595793:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768223.595797:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768223.595808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861537024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768223.595844:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768223.595863:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537024:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768224.171694:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768224.427595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768225.195655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768225.451607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:0.0:1775768226.219660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768226.475615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:1.0:1775768226.667777:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:1.0:1775768226.667835:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768226.667851:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775768226.667895:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:1.0:1775768226.667903:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775768226.667998:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768226] req@ffffa0e9d0540700 x1862023861536896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768226.668024:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861536896/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768226.668059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768226.668065:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768226.668069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861536896:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768226.668078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768226] req@ffffa0e9d0543800 x1862023861537024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768226.668090:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861537024/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768226.668097:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768226.668100:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768226.668102:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768226.668109:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768226] req@ffffa0e9d0540e00 x1862023861536768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768226.668121:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861536768/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768226.668128:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768226.668131:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768226.668133:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861536768:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775768227.243682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768227.499751:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768228.267638:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768228.523597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768228.715618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768228.715630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768228.715651:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768228.715661:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3957 00000100:00080000:0.0:1775768228.715670:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768228.715676:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768228.715704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861537152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768228.715729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768228.715731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768228.715732:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768228.715735:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3957 00000100:00080000:0.0:1775768228.715737:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768228.715739:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768228.715747:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768228.715756:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861537280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768228.715766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768228.715770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768228.715772:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775768228.715776:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861537152:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768228.715778:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3957 00000100:00080000:0.0:1775768228.715784:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768228.715788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768228.715793:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768228.715804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861537408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768228.715846:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768228.715891:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffffa0e9d09ddf80 x1862023861537152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768228.715910:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861537152/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768228.715934:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768228.715939:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768228.715943:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861537152:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768228.715981:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537280:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768228.716006:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768228.716019:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffffa0e9d09dca80 x1862023861537280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768228.716030:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861537280/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768228.716038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768228.716041:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768228.716044:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537280:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768228.716050:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768228.716069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768228.716098:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffffa0e9d09de680 x1862023861537408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768228.716109:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861537408/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768228.716115:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768228.716119:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768228.716121:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537408:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768229.291609:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768229.547599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768230.315703:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768230.571635:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768231.339689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768231.595579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768232.363709:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768232.619625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768233.387661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768233.643609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768233.835590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768233.835597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768233.835601:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768233.835609:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3962 00000100:00080000:0.0:1775768233.835618:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768233.835623:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768233.835658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861537536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768233.835680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768233.835682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768233.835683:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768233.835686:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3962 00000100:00080000:0.0:1775768233.835688:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768233.835690:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768233.835697:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768233.835704:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861537664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768233.835711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768233.835713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775768233.835730:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861537536:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768233.835731:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768233.835738:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3962 00000100:00080000:0.0:1775768233.835740:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768233.835743:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768233.835746:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768233.835755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861537792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768233.835788:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768233.835822:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768233] req@ffffa0e9d09de680 x1862023861537536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768233.835834:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861537536/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768233.835841:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768233.835844:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768233.835847:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861537536:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768233.835851:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537664:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768233.835863:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768233.835870:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768233] req@ffffa0e9d09dca80 x1862023861537664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768233.835876:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861537664/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768233.835881:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768233.835882:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768233.835884:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537664:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768233.835899:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768233.835908:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768233.835913:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768233] req@ffffa0e9d09ddf80 x1862023861537792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768233.835926:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861537792/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768233.835931:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768233.835932:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768233.835934:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861537792:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768234.411632:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768234.667594:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768235.435669:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768235.691598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768236.459669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768236.715666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768237.483669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768237.739622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768238.507661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768238.763588:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768238.955636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768238.955646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768238.955650:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768238.955660:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3967 00000100:00080000:3.0:1775768238.955671:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768238.955678:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768238.955716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861537920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768238.955750:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768238.955754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768238.955755:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768238.955758:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3967 00000100:00080000:3.0:1775768238.955780:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768238.955785:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768238.955794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768238.955796:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861537920:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775768238.955807:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861538048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768238.955815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768238.955817:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768238.955819:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768238.955821:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3967 00000100:00080000:3.0:1775768238.955823:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768238.955826:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768238.955829:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768238.955836:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861538176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768238.955857:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768238.955898:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffffa0e9d0542300 x1862023861537920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.955922:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861537920/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768238.955932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.955936:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768238.955940:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861537920:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768238.955961:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538048:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768238.955991:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768238.955999:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffffa0e9d0540380 x1862023861538048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.956007:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861538048/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768238.956025:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.956028:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768238.956030:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538048:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768238.956033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768238.956046:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768238.956055:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffffa0e9d0541c00 x1862023861538176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.956065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861538176/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768238.956098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.956102:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768238.956106:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538176:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768239.531668:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768239.787601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768240.555668:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768240.811568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768241.579706:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768241.835600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768242.603669:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768242.859612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768243.627678:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768243.883581:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768244.075619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768244.075627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768244.075632:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768244.075641:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3972 00000100:00080000:1.0:1775768244.075651:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768244.075656:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768244.075694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861538304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768244.075719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768244.075735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768244.075736:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768244.075740:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3972 00000100:00080000:1.0:1775768244.075744:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768244.075749:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768244.075760:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768244.075774:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861538432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768244.075786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768244.075789:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861538304:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768244.075791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768244.075796:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768244.075808:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3972 00000100:00080000:1.0:1775768244.075812:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768244.075814:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768244.075819:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768244.075831:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861538560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768244.075865:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768244.075909:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768244/real 1775768244] req@ffffa0e9c8dee680 x1862023861538304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768244.075921:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861538304/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768244.075928:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768244.075932:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768244.075934:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861538304:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768244.075968:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538432:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768244.075991:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768244.076000:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768244/real 1775768244] req@ffffa0e9c8ded880 x1862023861538432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768244.076007:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861538432/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768244.076012:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768244.076014:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768244.076016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538432:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768244.076020:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768244.076050:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768244.076057:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768244/real 1775768244] req@ffffa0e9c8deea00 x1862023861538560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768244.076064:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861538560/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768244.076068:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768244.076070:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768244.076072:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538560:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768244.651680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768244.907591:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768245.675665:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768245.931609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768246.699684:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768246.955642:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768247.723684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768247.979601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768248.747694:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768249.003740:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768249.195746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768249.195758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768249.195763:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768249.195774:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3977 00000100:00080000:3.0:1775768249.195787:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768249.195794:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768249.195838:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861538688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768249.195873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768249.195894:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768249.195896:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768249.195900:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3977 00000100:00080000:3.0:1775768249.195905:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768249.195909:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768249.195919:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768249.195933:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861538816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768249.195943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768249.195946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768249.195947:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768249.195951:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3977 00000100:00080000:3.0:1775768249.195954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768249.195957:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768249.195961:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768249.195970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861538944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768249.196033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861538688:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775768249.196103:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768249.196148:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffffa0e9d0542a00 x1862023861538688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768249.196165:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861538688/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768249.196174:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768249.196178:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768249.196182:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861538688:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768249.196190:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538816:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768249.196211:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768249.196222:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffffa0e9d0543b80 x1862023861538816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768249.196233:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861538816/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768249.196241:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768249.196243:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768249.196246:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538816:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768249.196252:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768249.196269:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768249.196279:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffffa0e9d0540000 x1862023861538944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768249.197426:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861538944/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768249.197435:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768249.197440:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768249.197443:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861538944:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768249.771712:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768250.027614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768250.795661:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768251.051596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768251.819664:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768252.075603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768252.843685:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768253.099580:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768253.867633:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768254.123600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768254.315566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768254.315576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768254.315581:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768254.315593:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3982 00000100:00080000:0.0:1775768254.315612:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768254.315620:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768254.315658:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861539072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768254.315686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768254.315688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768254.315690:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768254.315693:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3982 00000100:00080000:0.0:1775768254.315695:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768254.315710:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768254.315719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768254.315728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861539200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768254.315736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768254.315738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768254.315741:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775768254.315743:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861539072:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768254.315745:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3982 00000100:00080000:0.0:1775768254.315750:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768254.315753:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768254.315756:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768254.315763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861539328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768254.315813:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768254.315854:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffffa0e9d09ddc00 x1862023861539072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768254.315871:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861539072/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768254.315893:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.315897:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768254.315901:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861539072:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768254.315937:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539200:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768254.315958:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768254.315970:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffffa0e9d09dd180 x1862023861539200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768254.315981:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861539200/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768254.315990:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.315993:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768254.315996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539200:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768254.316002:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768254.316016:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768254.316024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffffa0e9d09dd500 x1862023861539328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768254.316035:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861539328/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768254.316042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.316045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768254.316048:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539328:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768254.891603:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768255.147617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768255.915669:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768256.171631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768256.939662:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768257.195598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768257.963709:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768258.219636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768258.987713:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768259.243648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768259.435620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768259.435628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768259.435632:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768259.435639:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3987 00000100:00080000:2.0:1775768259.435648:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768259.435653:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768259.435685:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861539456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768259.435711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768259.435714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768259.435733:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768259.435737:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3987 00000100:00080000:2.0:1775768259.435741:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768259.435744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768259.435754:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768259.435758:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861539456:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768259.435772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861539584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768259.435781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768259.435784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768259.435785:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768259.435788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3987 00000100:00080000:2.0:1775768259.435791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768259.435794:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768259.435798:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768259.435808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861539712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768259.435814:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768259.435853:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffffa0e9d0710380 x1862023861539456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.435867:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861539456/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768259.435884:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.435889:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768259.435891:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861539456:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768259.435908:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539584:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768259.435924:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768259.435931:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffffa0e9d0711180 x1862023861539584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.435938:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861539584/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768259.435943:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.435944:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768259.435946:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539584:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768259.435949:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539712:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768259.435966:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768259.435972:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffffa0e9d0713b80 x1862023861539712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.435978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861539712/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768259.435983:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.435985:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768259.435986:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539712:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768260.011705:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768260.267654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768261.035693:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768261.291654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768262.059672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768262.315612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768263.083674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768263.339650:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768264.107623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768264.363606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768264.555582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768264.555590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768264.555594:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768264.555603:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3993 00000100:00080000:1.0:1775768264.555615:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768264.555621:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768264.555657:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861539840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768264.555685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768264.555688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768264.555689:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768264.555691:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3993 00000100:00080000:1.0:1775768264.555693:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768264.555713:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768264.555725:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768264.555733:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861539840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768264.555740:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861539968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768264.555753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768264.555757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768264.555759:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768264.555762:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3993 00000100:00080000:1.0:1775768264.555765:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768264.555768:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768264.555772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768264.555781:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861540096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768264.555798:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768264.555859:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffffa0e9c8deca80 x1862023861539840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768264.555877:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861539840/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768264.555885:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768264.555888:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768264.555891:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861539840:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768264.555910:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539968:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768264.555926:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768264.555933:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffffa0e9c8dedf80 x1862023861539968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768264.555941:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861539968/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768264.555946:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768264.555948:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768264.555949:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861539968:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768264.555952:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540096:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768264.555961:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768264.555966:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffffa0e9c8dece00 x1862023861540096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768264.555973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861540096/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768264.555985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768264.555987:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768264.555989:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540096:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768265.131747:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768265.387707:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768266.155623:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768266.411617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768267.179644:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768267.435656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768268.203634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768268.459579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768269.227689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768269.483627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768269.675620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768269.675631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768269.675635:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768269.675645:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 3998 00000100:00080000:0.0:1775768269.675655:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768269.675662:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768269.675700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861540224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768269.675735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768269.675738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768269.675740:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768269.675745:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3998 00000100:00080000:0.0:1775768269.675749:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768269.675753:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768269.675760:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861540224:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775768269.675763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768269.675795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861540352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768269.675823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768269.675826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768269.675828:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775768269.675832:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:0.0:1775768269.675835:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 3998 00000100:00080000:0.0:1775768269.675841:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768269.675845:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768269.675850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768269.675862:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861540480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768269.675882:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffffa0e9d09df800 x1862023861540224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768269.675900:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861540224/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768269.675910:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768269.675913:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768269.675917:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861540224:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768269.675945:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540352:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768269.675965:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768269.675975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffffa0e9d09dc700 x1862023861540352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768269.675986:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861540352/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768269.675992:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768269.675995:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768269.675998:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540352:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768269.676003:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540480:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768269.676018:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768269.676026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffffa0e9d09dd880 x1862023861540480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768269.676036:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861540480/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768269.676042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768269.676045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768269.676048:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540480:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768270.251676:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768270.507586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768271.275616:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768271.531608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768272.299688:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768272.555656:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768273.323612:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768273.579597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768274.347695:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768274.603625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768274.795605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768274.795615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768274.795620:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768274.795630:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4003 00000100:00080000:0.0:1775768274.795642:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768274.795647:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768274.795682:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861540608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768274.795708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768274.795711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768274.795723:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768274.795736:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4003 00000100:00080000:0.0:1775768274.795739:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768274.795743:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768274.795752:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768274.795761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861540736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768274.795769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768274.795773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775768274.795774:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861540608:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768274.795775:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768274.795784:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4003 00000100:00080000:0.0:1775768274.795788:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768274.795793:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768274.795800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768274.795812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861540864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768274.795866:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768274.795910:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffffa0e9d09ded80 x1862023861540608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768274.795925:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861540608/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768274.795932:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768274.795935:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768274.795938:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861540608:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768274.795966:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540736:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768274.795982:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768274.795990:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffffa0e9d09dfb80 x1862023861540736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768274.795998:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861540736/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768274.796003:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768274.796005:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768274.796007:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540736:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768274.796010:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540864:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768274.796019:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768274.796023:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffffa0e9d09df100 x1862023861540864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768274.796042:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861540864/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768274.796049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768274.796052:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768274.796055:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861540864:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768275.371675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768275.627643:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768276.395737:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768276.651598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768277.419682:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768277.675644:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768278.443632:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768278.699597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768279.467639:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768279.723666:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768279.915630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768279.915640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768279.915646:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768279.915657:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4008 00000100:00080000:0.0:1775768279.915670:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768279.915708:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768279.915750:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861540992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768279.915777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768279.915781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768279.915782:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768279.915786:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4008 00000100:00080000:0.0:1775768279.915791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768279.915796:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768279.915808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768279.915824:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861541120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768279.915825:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861540992:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768279.915848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768279.915852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768279.915853:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768279.915860:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4008 00000100:00080000:0.0:1775768279.915863:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768279.915867:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768279.915873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768279.915884:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861541248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768279.915904:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768279.915944:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffffa0e9d09dce00 x1862023861540992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768279.915958:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861540992/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768279.915965:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768279.915968:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768279.915971:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861540992:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768279.915990:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541120:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768279.916003:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768279.916010:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffffa0e9d09dc380 x1862023861541120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768279.916017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861541120/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768279.916031:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768279.916033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768279.916035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541120:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768279.916039:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541248:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768279.916051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768279.916057:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffffa0e9d09dc000 x1862023861541248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768279.916071:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861541248/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768279.916076:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768279.916078:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768279.916080:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541248:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768280.491703:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768280.747641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768281.515682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768281.771671:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768282.539667:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768282.795634:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768283.563723:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768283.819596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768284.587675:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768284.843625:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768285.035665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768285.035673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768285.035678:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768285.035687:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4013 00000100:00080000:2.0:1775768285.035699:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768285.035704:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768285.035743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861541376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768285.035772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768285.035776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768285.035778:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768285.035782:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4013 00000100:00080000:2.0:1775768285.035786:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768285.035789:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768285.035799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768285.035810:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861541504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768285.035825:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861541376:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768285.035843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768285.035847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768285.035848:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768285.035854:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4013 00000100:00080000:2.0:1775768285.035859:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768285.035863:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768285.035869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768285.035882:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861541632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768285.035901:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768285.035950:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768285/real 1775768285] req@ffffa0e9d0711c00 x1862023861541376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768285.035969:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861541376/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768285.035980:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768285.035984:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768285.035994:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861541376:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768285.036021:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541504:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768285.036044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768285.036057:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768285/real 1775768285] req@ffffa0e9d0710000 x1862023861541504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768285.036070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861541504/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768285.036077:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768285.036081:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768285.036084:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541504:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768285.036090:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541632:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768285.036116:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768285.036126:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768285/real 1775768285] req@ffffa0e9d0711f80 x1862023861541632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768285.036138:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861541632/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768285.036143:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768285.036145:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768285.036147:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541632:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768285.611659:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768285.867649:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768286.635729:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768286.891654:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768287.659703:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768287.915616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768288.683779:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768288.939655:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768289.707737:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768289.963570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768290.155646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768290.155653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768290.155657:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768290.155665:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4018 00000100:00080000:2.0:1775768290.155674:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768290.155679:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768290.155716:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861541760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768290.155767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768290.155770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768290.155772:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768290.155776:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4018 00000100:00080000:2.0:1775768290.155779:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768290.155783:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768290.155791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768290.155801:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861541888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768290.155808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768290.155810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768290.155811:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768290.155814:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4018 00000100:00080000:2.0:1775768290.155816:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768290.155818:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768290.155821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768290.155826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861542016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768290.155856:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861541760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768290.155984:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541888:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768290.156020:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542016:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768290.731653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768290.987619:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768291.755698:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768292.011579:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768292.779673:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768293.035633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775768293.227800:0:2391:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775768293.227846:0:2391:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768293.227864:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768293.227917:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768293.227926:0:2391:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775768293.227966:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768290/real 1775768293] req@ffffa0e9d0712d80 x1862023861541760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768293.227995:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861541760/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768293.228006:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768293.228011:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768293.228015:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861541760:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768293.228024:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768290/real 1775768293] req@ffffa0e9d0710a80 x1862023861541888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768293.228034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861541888/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768293.228043:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768293.228046:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768293.228049:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861541888:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768293.228055:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768290/real 1775768293] req@ffffa0e9d0710700 x1862023861542016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768293.228065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861542016/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768293.228086:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768293.228090:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768293.228094:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542016:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768293.803678:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768294.059603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768294.827661:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768295.083607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768295.275622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768295.275631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768295.275648:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768295.275656:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4023 00000100:00080000:0.0:1775768295.275670:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768295.275677:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768295.275710:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861542144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768295.275740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768295.275743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768295.275745:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768295.275749:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4023 00000100:00080000:0.0:1775768295.275753:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768295.275757:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768295.275770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768295.275804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861542272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768295.275813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768295.275816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768295.275818:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768295.275821:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4023 00000100:00080000:0.0:1775768295.275825:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768295.275829:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768295.275845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768295.275854:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861542144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768295.275861:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861542400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768295.275919:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768295.275981:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffffa0e9d09de300 x1862023861542144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768295.276000:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861542144/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768295.276010:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768295.276015:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768295.276018:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861542144:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768295.276025:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542272:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768295.276044:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768295.276055:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffffa0e9d09dea00 x1862023861542272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768295.276065:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861542272/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768295.276074:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768295.276077:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768295.276086:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542272:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768295.276108:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542400:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768295.276124:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768295.276133:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffffa0e9d09df480 x1862023861542400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768295.276143:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861542400/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768295.276151:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768295.276153:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768295.276156:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542400:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768295.851661:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768296.107607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768296.875653:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768297.131583:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768297.899624:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768298.155608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768298.923667:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768299.179614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768299.947689:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768300.203603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768300.395628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768300.395641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768300.395646:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768300.395656:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4028 00000100:00080000:2.0:1775768300.395666:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768300.395672:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768300.395723:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861542528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768300.395750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768300.395754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768300.395756:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768300.395779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4028 00000100:00080000:2.0:1775768300.395784:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768300.395789:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768300.395800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768300.395801:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861542528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768300.395817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861542656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768300.395829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768300.395833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768300.395835:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768300.395839:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4028 00000100:00080000:2.0:1775768300.395843:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768300.395846:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768300.395851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768300.395864:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861542784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768300.395869:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768300.395920:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768300] req@ffffa0e9d0710e00 x1862023861542528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768300.395934:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861542528/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768300.395940:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768300.395943:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768300.395946:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861542528:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768300.395964:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542656:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768300.395987:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768300.395995:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768300] req@ffffa0e9d0712680 x1862023861542656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768300.396002:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861542656/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768300.396006:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768300.396008:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768300.396010:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542656:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768300.396013:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542784:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768300.396022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768300.396026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768300] req@ffffa0e9d0711500 x1862023861542784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768300.396032:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861542784/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768300.396036:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768300.396038:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768300.396039:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861542784:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768300.971623:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768301.227604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768301.995699:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768302.251601:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768303.019665:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768303.275600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768304.043671:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768304.299602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768305.067640:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768305.323603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768305.515590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768305.515599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768305.515605:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768305.515614:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4034 00000100:00080000:0.0:1775768305.515626:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768305.515632:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768305.515677:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df480 x1862023861542912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768305.515708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768305.515712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768305.515714:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768305.515717:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4034 00000100:00080000:0.0:1775768305.515721:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768305.515725:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768305.515734:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768305.515751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dea00 x1862023861543040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768305.515761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768305.515761:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861542912:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768305.515769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768305.515771:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768305.515776:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4034 00000100:00080000:0.0:1775768305.515780:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768305.515785:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768305.515813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768305.515825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de300 x1862023861543168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768305.515835:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768305.515892:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffffa0e9d09df480 x1862023861542912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768305.515920:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df480 x1862023861542912/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768305.515931:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768305.515936:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768305.515939:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861542912:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768305.515967:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543040:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768305.515991:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768305.516001:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffffa0e9d09dea00 x1862023861543040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768305.516010:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dea00 x1862023861543040/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768305.516015:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768305.516017:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768305.516019:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543040:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768305.516023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543168:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768305.516035:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768305.516040:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffffa0e9d09de300 x1862023861543168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768305.516060:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de300 x1862023861543168/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768305.516080:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768305.516084:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768305.516087:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543168:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768306.091650:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768306.347586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768307.115630:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768307.371586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768308.139665:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768308.395606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768309.163676:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768309.419571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768310.187638:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768310.443568:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768310.635587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768310.635595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768310.635604:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768310.635613:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4039 00000100:00080000:0.0:1775768310.635622:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768310.635627:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768310.635659:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc000 x1862023861543296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768310.635680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768310.635682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768310.635683:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768310.635686:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4039 00000100:00080000:0.0:1775768310.635688:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768310.635690:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768310.635697:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768310.635705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc380 x1862023861543424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768310.635712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768310.635714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768310.635715:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768310.635717:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4039 00000100:00100000:1.0:1775768310.635723:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861543296:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768310.635734:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768310.635738:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768310.635741:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768310.635749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dce00 x1862023861543552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768310.635778:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768310.635812:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffffa0e9d09dc000 x1862023861543296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768310.635824:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc000 x1862023861543296/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768310.635830:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768310.635833:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768310.635836:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861543296:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768310.635840:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543424:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768310.635852:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768310.635858:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffffa0e9d09dc380 x1862023861543424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768310.635865:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc380 x1862023861543424/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768310.635870:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768310.635877:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768310.635879:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543424:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768310.635896:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543552:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768310.635908:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768310.635913:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffffa0e9d09dce00 x1862023861543552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768310.635920:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dce00 x1862023861543552/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768310.635924:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768310.635926:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768310.635927:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543552:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768311.211674:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768311.467554:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768312.235646:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768312.491597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768313.259651:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768313.515596:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768314.283681:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768314.539617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768315.307693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768315.563617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768315.755609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768315.755617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768315.755622:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768315.755632:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4044 00000100:00080000:0.0:1775768315.755645:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768315.755652:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768315.755695:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df100 x1862023861543680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768315.755723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768315.755727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768315.755728:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768315.755731:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4044 00000100:00080000:0.0:1775768315.755733:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768315.755736:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768315.755745:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768315.755756:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dfb80 x1862023861543808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768315.755772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768315.755783:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861543680:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768315.755793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768315.755795:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768315.755801:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4044 00000100:00080000:0.0:1775768315.755804:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768315.755809:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768315.755815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768315.755828:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ded80 x1862023861543936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768315.755869:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768315.755920:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffffa0e9d09df100 x1862023861543680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768315.755937:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df100 x1862023861543680/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768315.755945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768315.755948:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768315.755951:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861543680:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768315.755971:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543808:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768315.755994:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768315.756003:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffffa0e9d09dfb80 x1862023861543808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768315.756011:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dfb80 x1862023861543808/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768315.756016:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768315.756018:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768315.756020:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543808:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768315.756023:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543936:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768315.756032:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768315.756037:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffffa0e9d09ded80 x1862023861543936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768315.756044:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ded80 x1862023861543936/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768315.756049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768315.756051:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768315.756053:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861543936:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768316.331695:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768316.587611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768317.355664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768317.611609:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768318.379797:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768318.635598:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768319.403675:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768319.659771:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768320.427646:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768320.683605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768320.875604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768320.875613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768320.875618:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768320.875627:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4049 00000100:00080000:2.0:1775768320.875638:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768320.875645:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768320.875683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861544064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768320.875731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768320.875735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768320.875736:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768320.875740:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4049 00000100:00080000:2.0:1775768320.875744:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768320.875748:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768320.875757:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768320.875769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861544192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768320.875777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768320.875779:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861544064:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768320.875780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768320.875781:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768320.875784:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4049 00000100:00080000:2.0:1775768320.875787:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768320.875790:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768320.875794:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768320.875802:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861544320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768320.875837:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768320.875877:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffffa0e9d0713480 x1862023861544064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768320.875901:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861544064/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768320.875910:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768320.875914:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768320.875918:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861544064:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768320.875941:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544192:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768320.875962:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768320.875974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffffa0e9d0711880 x1862023861544192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768320.875986:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861544192/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768320.875993:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768320.875996:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768320.875999:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544192:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768320.876004:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544320:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768320.876022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768320.876032:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffffa0e9d0712a00 x1862023861544320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768320.876050:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861544320/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768320.876064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768320.876066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768320.876068:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544320:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768321.451700:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768321.707611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768322.475653:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768322.731592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768323.499689:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768323.755621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768324.523643:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768324.779610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768325.547709:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768325.803600:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768325.995627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768325.995638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768325.995642:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768325.995652:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4054 00000100:00080000:2.0:1775768325.995662:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768325.995668:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768325.995707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713100 x1862023861544448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768325.995737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768325.995741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768325.995743:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768325.995746:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4054 00000100:00080000:2.0:1775768325.995750:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768325.995754:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768325.995763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768325.995776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712300 x1862023861544576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768325.995781:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861544448:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768325.995794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768325.995797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768325.995799:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768325.995804:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4054 00000100:00080000:2.0:1775768325.995808:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768325.995830:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000800:00000100:1.0:1775768325.995836:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000008:00100000:2.0:1775768325.995836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768325.995858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712a00 x1862023861544704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768325.995879:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768325/real 1775768325] req@ffffa0e9d0713100 x1862023861544448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768325.995894:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713100 x1862023861544448/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768325.995902:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768325.995905:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768325.995907:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861544448:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768325.995924:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544576:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768325.995936:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768325.995943:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768325/real 1775768325] req@ffffa0e9d0712300 x1862023861544576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768325.995950:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712300 x1862023861544576/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768325.995954:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768325.995963:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768325.995965:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544576:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768325.995969:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544704:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768325.995980:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768325.995986:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768325/real 1775768325] req@ffffa0e9d0712a00 x1862023861544704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768325.995992:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712a00 x1862023861544704/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768325.995996:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768325.995998:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768325.996000:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544704:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768326.571610:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768326.827608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768327.595660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768327.851604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768328.619682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768328.875582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768329.643614:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768329.899610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768330.667695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768330.923578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768331.115606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768331.115618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768331.115623:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768331.115638:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4059 00000100:00080000:3.0:1775768331.115648:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768331.115676:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768331.115707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861544832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768331.115738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768331.115741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768331.115743:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768331.115747:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4059 00000100:00080000:3.0:1775768331.115750:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768331.115753:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768331.115762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768331.115773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861544832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775768331.115774:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861544960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768331.115786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768331.115788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768331.115789:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768331.115792:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4059 00000100:00080000:3.0:1775768331.115794:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768331.115797:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768331.115800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768331.115812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861545088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768331.115831:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768331.115873:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768331/real 1775768331] req@ffffa0e9d0542680 x1862023861544832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768331.115886:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861544832/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768331.115895:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768331.115898:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768331.115900:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861544832:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768331.115916:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544960:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768331.115937:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768331.115944:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768331/real 1775768331] req@ffffa0e9d0541500 x1862023861544960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768331.115951:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861544960/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768331.115956:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768331.115958:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768331.115960:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861544960:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768331.115963:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545088:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768331.115972:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768331.115977:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768331/real 1775768331] req@ffffa0e9d0541880 x1862023861545088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768331.115983:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861545088/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768331.115987:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768331.115989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768331.115991:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545088:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768331.691712:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768331.947603:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768332.715700:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768332.971557:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768333.739625:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768333.995615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768334.763696:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768335.019620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768335.787638:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768336.043564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768336.235622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768336.235633:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768336.235638:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768336.235648:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4064 00000100:00080000:0.0:1775768336.235657:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768336.235663:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768336.235699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861545216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768336.235725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768336.235728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768336.235730:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768336.235734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4064 00000100:00080000:0.0:1775768336.235738:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768336.235743:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768336.235752:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768336.235763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861545344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768336.235767:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861545216:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768336.235788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768336.235790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768336.235791:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768336.235795:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4064 00000100:00080000:0.0:1775768336.235798:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768336.235800:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768336.235804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768336.235814:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861545472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768336.235823:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768336.235859:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768336/real 1775768336] req@ffffa0e9d09dd880 x1862023861545216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768336.235878:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861545216/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768336.235885:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768336.235888:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768336.235890:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861545216:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768336.235907:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545344:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768336.235921:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768336.235929:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768336/real 1775768336] req@ffffa0e9d09dc700 x1862023861545344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768336.235936:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861545344/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768336.235941:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768336.235943:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768336.235945:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545344:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768336.235948:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545472:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768336.235956:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768336.235961:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768336/real 1775768336] req@ffffa0e9d09df800 x1862023861545472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768336.235967:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861545472/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768336.235979:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768336.235981:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768336.235983:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545472:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768336.811840:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768337.067604:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768337.835686:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768338.091575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768338.859655:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768339.115622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768339.883660:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768340.140134:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768340.907696:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768341.163640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768341.355732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768341.355743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768341.355768:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768341.355779:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4069 00000100:00080000:3.0:1775768341.355790:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768341.355796:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768341.355833:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541880 x1862023861545600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768341.355866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768341.355870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768341.355872:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768341.355876:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4069 00000100:00080000:3.0:1775768341.355880:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768341.355883:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768341.355893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768341.355907:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541500 x1862023861545728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768341.355910:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861545600:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775768341.355924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768341.355927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768341.355929:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768341.355934:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4069 00000100:00080000:3.0:1775768341.355938:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768341.355942:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768341.355948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775768341.355967:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:3.0:1775768341.355968:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542680 x1862023861545856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768341.356020:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768341/real 1775768341] req@ffffa0e9d0541880 x1862023861545600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768341.356039:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541880 x1862023861545600/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768341.356049:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768341.356055:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768341.356060:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861545600:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768341.356099:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545728:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768341.356116:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768341.356125:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768341/real 1775768341] req@ffffa0e9d0541500 x1862023861545728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768341.356133:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541500 x1862023861545728/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768341.356138:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768341.356145:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768341.356147:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545728:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768341.356151:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545856:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768341.356170:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768341.356181:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768341/real 1775768341] req@ffffa0e9d0542680 x1862023861545856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768341.356193:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542680 x1862023861545856/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768341.356199:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768341.356202:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768341.356204:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861545856:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768341.931732:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768342.187891:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768342.955668:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768343.211617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768343.979677:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768344.235621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768345.005127:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768345.259637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768346.027637:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768346.283960:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768346.475635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768346.475647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768346.475651:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768346.475662:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4074 00000100:00080000:3.0:1775768346.475673:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768346.475680:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768346.475723:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540000 x1862023861545984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768346.475811:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861545984:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775768346.475885:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:3.0:1775768346.475928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768346.475932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768346.475934:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768346.475939:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4074 00000100:00080000:3.0:1775768346.475943:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768346.475947:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768346.475957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768346.475969:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543b80 x1862023861546112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768346.476114:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768346/real 1775768346] req@ffffa0e9d0540000 x1862023861545984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768346.476131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540000 x1862023861545984/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768346.476141:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768346.476146:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768346.476149:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861545984:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768346.476170:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546112:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768346.476212:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768346.476225:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768346/real 1775768346] req@ffffa0e9d0543b80 x1862023861546112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768346.476237:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543b80 x1862023861546112/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768346.476245:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768346.476248:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768346.476251:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546112:192.168.202.104@tcp:38:kworker.0 00000100:00080000:3.0:1775768346.476374:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768346.476379:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768346.476381:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768346.476385:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4074 00000100:00080000:3.0:1775768346.476389:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768346.476398:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768346.476410:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768346.476422:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542a00 x1862023861546240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768346.476468:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546240:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768346.476517:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768346.476642:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768346/real 1775768346] req@ffffa0e9d0542a00 x1862023861546240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768346.476654:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542a00 x1862023861546240/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768346.476662:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768346.476665:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768346.476668:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546240:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768347.053144:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768347.307631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768348.075748:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768348.331614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768349.103075:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768349.359300:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768350.123763:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768350.380932:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768351.147643:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768351.403631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768351.595623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768351.595634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768351.595639:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768351.595650:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4080 00000100:00080000:2.0:1775768351.595661:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768351.595668:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768351.595709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711880 x1862023861546368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768351.595737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768351.595741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768351.595743:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768351.595747:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4080 00000100:00080000:2.0:1775768351.595750:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768351.595773:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768351.595783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768351.595788:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861546368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:2.0:1775768351.595799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713480 x1862023861546496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768351.595810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768351.595814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768351.595816:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768351.595819:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4080 00000100:00080000:2.0:1775768351.595823:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768351.595827:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768351.595832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768351.595843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711500 x1862023861546624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768351.595855:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768351.595907:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768351/real 1775768351] req@ffffa0e9d0711880 x1862023861546368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768351.595934:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711880 x1862023861546368/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768351.595945:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768351.595949:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768351.595953:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861546368:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768351.595999:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546496:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768351.596024:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768351.596036:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768351/real 1775768351] req@ffffa0e9d0713480 x1862023861546496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768351.596048:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713480 x1862023861546496/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768351.596056:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768351.596058:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768351.596062:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546496:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768351.596068:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546624:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768351.596086:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768351.596096:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768351/real 1775768351] req@ffffa0e9d0711500 x1862023861546624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768351.596107:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711500 x1862023861546624/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768351.596125:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768351.596128:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768351.596131:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546624:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768352.172476:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768352.427648:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768353.196019:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768353.451878:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768354.219774:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768354.475611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768355.244081:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768355.499606:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768356.267810:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768356.523633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768356.715643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768356.715653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768356.715657:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768356.715667:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4085 00000100:00080000:2.0:1775768356.715678:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768356.715705:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768356.715738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712680 x1862023861546752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768356.715772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768356.715776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768356.715778:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768356.715782:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4085 00000100:00080000:2.0:1775768356.715785:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768356.715789:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768356.715797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768356.715807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710e00 x1862023861546880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768356.715816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768356.715819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768356.715820:0:36:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768356.715823:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4085 00000100:00080000:2.0:1775768356.715826:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768356.715829:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768356.715833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768356.715840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861547008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768356.716032:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861546752:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768356.716121:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546880:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768356.716144:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547008:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768357.291738:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768357.547819:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768358.315770:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768358.571629:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768359.339674:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768359.595620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:3.0:1775768359.787810:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:3.0:1775768359.787857:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775768359.787874:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775768359.788133:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768356/real 1775768359] req@ffffa0e9d0712680 x1862023861546752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768381 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768359.788163:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712680 x1862023861546752/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768381 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768359.788177:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768359.788182:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768359.788186:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861546752:192.168.202.104@tcp:250:kworker.0 00000800:00000100:3.0:1775768359.789824:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:3.0:1775768359.789851:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:1.0:1775768359.789902:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768356/real 1775768359] req@ffffa0e9d0710e00 x1862023861546880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768359.789933:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710e00 x1862023861546880/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768359.789964:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768359.789967:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768359.789970:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861546880:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768359.789975:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768356/real 1775768359] req@ffffa0e9d0710700 x1862023861547008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768359.789983:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861547008/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768359.789988:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768359.789990:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768359.789992:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547008:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768360.363943:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768360.619616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768361.387816:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768361.643620:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768361.835624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768361.835635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768361.835648:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768361.835659:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4090 00000100:00080000:2.0:1775768361.835674:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768361.835681:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768361.835721:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861547136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768361.835749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768361.835752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768361.835754:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768361.835758:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4090 00000100:00080000:2.0:1775768361.835762:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768361.835765:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768361.835774:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768361.835787:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861547264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768361.835798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768361.835800:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861547136:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768361.835802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768361.835822:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768361.835826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4090 00000100:00080000:2.0:1775768361.835830:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768361.835833:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768361.835838:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768361.835859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861547392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768361.835865:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768361.835940:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768361/real 1775768361] req@ffffa0e9d0710a80 x1862023861547136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768361.835959:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861547136/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768361.835970:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768361.835973:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768361.835977:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861547136:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768361.836000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547264:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768361.836031:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768361.836043:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768361/real 1775768361] req@ffffa0e9d0712d80 x1862023861547264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768361.836055:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861547264/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768361.836063:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768361.836066:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768361.836069:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547264:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768361.836075:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547392:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768361.836094:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768361.836104:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768361/real 1775768361] req@ffffa0e9d0711f80 x1862023861547392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768361.836114:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861547392/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768361.836121:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768361.836124:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768361.836127:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547392:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768362.411685:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768362.667610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768363.435627:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768363.691612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768364.459823:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768364.715784:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768365.484617:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768365.739578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768366.507688:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768366.763711:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768366.955734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768366.955753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768366.955761:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768366.955772:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4095 00000100:00080000:0.0:1775768366.955784:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768366.955791:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768366.955831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861547520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768366.955865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768366.955869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768366.955870:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768366.955874:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4095 00000100:00080000:0.0:1775768366.955878:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768366.955882:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768366.955899:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861547520:192.168.202.104@tcp:250:kworker.0 00000008:00100000:0.0:1775768366.955908:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768366.955919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861547648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768366.955927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768366.955930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768366.955931:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768366.955934:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4095 00000100:00080000:0.0:1775768366.955946:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768366.955949:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768366.955953:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768366.955965:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861547776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768366.955986:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768366.956071:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768366/real 1775768366] req@ffffa0e9d09dd500 x1862023861547520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768366.956105:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861547520/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768366.956124:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768366.956127:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768366.956129:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861547520:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768366.956148:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547648:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768366.956173:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768366.956212:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768366/real 1775768366] req@ffffa0e9d09dd180 x1862023861547648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768366.956225:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861547648/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768366.956230:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768366.956232:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768366.956234:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547648:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768366.956238:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547776:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768366.956251:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768366.956285:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768366/real 1775768366] req@ffffa0e9d09ddc00 x1862023861547776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768366.956296:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861547776/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768366.956332:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768366.956334:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768366.956336:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861547776:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768367.531684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768367.787624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768368.555673:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768368.811614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768369.579658:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768369.835633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768370.603651:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768370.859575:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768371.627707:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768371.883597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768372.075616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768372.075626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768372.075631:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768372.075641:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4100 00000100:00080000:1.0:1775768372.075651:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768372.075657:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768372.075695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861547904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768372.075723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768372.075727:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768372.075728:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768372.075732:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4100 00000100:00080000:1.0:1775768372.075737:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768372.075741:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768372.075751:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768372.075754:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861547904:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768372.075766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861548032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768372.075778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768372.075806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768372.075808:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768372.075812:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4100 00000100:00080000:1.0:1775768372.075817:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768372.075822:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:1.0:1775768372.075825:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768372.075831:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768372.075844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861548160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768372.075876:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768372/real 1775768372] req@ffffa0e9c8dec700 x1862023861547904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768372.075895:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861547904/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768372.075904:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768372.075908:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768372.075912:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861547904:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768372.075939:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548032:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768372.075970:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768372.075983:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768372/real 1775768372] req@ffffa0e9c8defb80 x1862023861548032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768372.076003:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861548032/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768372.076010:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768372.076013:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768372.076016:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548032:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768372.076021:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548160:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768372.076040:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768372.076051:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768372/real 1775768372] req@ffffa0e9c8def100 x1862023861548160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768372.076061:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861548160/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768372.076066:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768372.076068:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768372.076070:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548160:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768372.651680:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768372.907587:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768373.675717:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768373.931599:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768374.699672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768374.955553:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768375.723748:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768375.979612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768376.747719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768377.003630:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768377.195632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768377.195644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768377.195648:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768377.195656:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4105 00000100:00080000:1.0:1775768377.195666:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768377.195671:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768377.195707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def100 x1862023861548288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768377.195731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768377.195733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768377.195757:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768377.195761:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4105 00000100:00080000:1.0:1775768377.195765:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768377.195769:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768377.195780:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768377.195802:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861548288:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768377.195810:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8defb80 x1862023861548416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768377.195818:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768377.195820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768377.195822:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768377.195826:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4105 00000100:00080000:1.0:1775768377.195830:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768377.195834:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768377.195839:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768377.195849:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec700 x1862023861548544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768377.195865:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768377.195908:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768377/real 1775768377] req@ffffa0e9c8def100 x1862023861548288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768377.195939:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def100 x1862023861548288/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768377.195960:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768377.195964:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768377.195968:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861548288:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768377.195991:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548416:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768377.196011:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768377.196022:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768377/real 1775768377] req@ffffa0e9c8defb80 x1862023861548416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768377.196034:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8defb80 x1862023861548416/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768377.196042:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768377.196045:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768377.196048:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548416:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768377.196054:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548544:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768377.196069:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768377.196078:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768377/real 1775768377] req@ffffa0e9c8dec700 x1862023861548544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768377.196088:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec700 x1862023861548544/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768377.196107:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768377.196110:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768377.196113:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548544:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768377.773319:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768378.027633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768378.795771:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768379.051627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768379.819708:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768380.075682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768380.843867:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768381.099651:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768381.867995:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768382.123768:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768382.315588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768382.315598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768382.315603:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768382.315613:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4110 00000100:00080000:2.0:1775768382.315623:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768382.315630:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768382.315665:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861548672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768382.315723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768382.315727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768382.315729:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768382.315733:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4110 00000100:00080000:2.0:1775768382.315737:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768382.315742:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768382.315744:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861548672:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775768382.315751:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768382.315767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861548800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768382.315792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768382.315796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768382.315797:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768382.315801:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4110 00000800:00000100:0.0:1775768382.315806:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00080000:2.0:1775768382.315807:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768382.315812:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768382.315817:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768382.315828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861548928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768382.315862:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768382/real 1775768382] req@ffffa0e9d0710000 x1862023861548672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768382.315880:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861548672/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768382.315890:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768382.315894:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768382.315898:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861548672:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768382.315917:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548800:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768382.315935:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768382.315945:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768382/real 1775768382] req@ffffa0e9d0711c00 x1862023861548800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768382.315956:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861548800/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768382.315963:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768382.315966:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768382.315968:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548800:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768382.315980:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548928:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768382.315997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768382.316005:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768382/real 1775768382] req@ffffa0e9d0713b80 x1862023861548928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768382.316016:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861548928/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768382.316023:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768382.316025:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768382.316028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861548928:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768382.891737:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768383.147739:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768383.915762:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768384.171564:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768384.939712:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768385.195641:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768385.964519:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768386.219777:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768386.987745:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768387.243626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768387.435631:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768387.435649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768387.435654:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768387.435662:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4115 00000100:00080000:1.0:1775768387.435671:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768387.435676:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768387.435715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dece00 x1862023861549056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768387.435745:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768387.435766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768387.435768:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768387.435771:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4115 00000100:00080000:1.0:1775768387.435775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768387.435778:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768387.435786:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861549056:192.168.202.104@tcp:250:kworker.0 00000008:00100000:1.0:1775768387.435787:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768387.435801:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedf80 x1862023861549184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768387.435809:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768387.435811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768387.435813:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768387.435815:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4115 00000100:00080000:1.0:1775768387.435817:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768387.435819:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768387.435822:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768387.435828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deca80 x1862023861549312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768387.435841:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768387.435881:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768387/real 1775768387] req@ffffa0e9c8dece00 x1862023861549056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768387.435894:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dece00 x1862023861549056/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768387.435904:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768387.435915:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768387.435919:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861549056:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768387.435941:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549184:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768387.435974:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768387.435987:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768387/real 1775768387] req@ffffa0e9c8dedf80 x1862023861549184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768387.436012:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedf80 x1862023861549184/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768387.436020:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768387.436023:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768387.436026:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549184:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768387.436031:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549312:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768387.436051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768387.436061:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768387/real 1775768387] req@ffffa0e9c8deca80 x1862023861549312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768387.436072:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deca80 x1862023861549312/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768387.436080:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768387.436082:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768387.436095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549312:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768388.011769:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768388.267653:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768389.035760:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768389.291607:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768390.059714:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768390.315636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768391.083742:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768391.339724:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768392.107732:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768392.363582:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768392.555668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768392.555679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768392.555684:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768392.555694:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4121 00000100:00080000:3.0:1775768392.555704:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768392.555711:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768392.555750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541c00 x1862023861549440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768392.555788:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768392.555792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768392.555793:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768392.555797:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4121 00000100:00080000:3.0:1775768392.555801:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768392.555805:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768392.555813:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768392.555825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540380 x1862023861549568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768392.555854:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768392.555857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768392.555858:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768392.555862:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4121 00000100:00080000:3.0:1775768392.555866:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768392.555869:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768392.555874:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768392.555883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542300 x1862023861549696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768392.555989:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861549440:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775768392.556062:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768392.556114:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768392/real 1775768392] req@ffffa0e9d0541c00 x1862023861549440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768392.556133:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541c00 x1862023861549440/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768392.556156:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768392.556160:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768392.556164:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861549440:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768392.556171:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549568:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768392.556207:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768392.556219:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768392/real 1775768392] req@ffffa0e9d0540380 x1862023861549568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768392.556230:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540380 x1862023861549568/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768392.556238:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768392.556241:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768392.556244:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549568:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768392.556250:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549696:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768392.556268:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768392.556278:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768392/real 1775768392] req@ffffa0e9d0542300 x1862023861549696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768392.556289:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542300 x1862023861549696/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768392.556297:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768392.556579:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768392.556584:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549696:192.168.202.104@tcp:38:kworker.0 00010000:00010000:1.0:1775768393.131858:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768393.387633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768394.155787:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768394.411759:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768395.179719:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768395.435628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:1.0:1775768396.203696:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768396.459682:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768397.227764:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768397.483647:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768397.675629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768397.675637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768397.675642:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768397.675650:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4126 00000100:00080000:3.0:1775768397.675660:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768397.675666:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768397.675703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540e00 x1862023861549824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768397.675736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768397.675739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768397.675741:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768397.675744:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4126 00000100:00080000:3.0:1775768397.675748:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768397.675752:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768397.675776:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768397.675784:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861549824:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775768397.675787:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543800 x1862023861549952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768397.675797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768397.675799:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768397.675800:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768397.675803:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4126 00000100:00080000:3.0:1775768397.675805:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768397.675808:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768397.675810:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768397.675818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540700 x1862023861550080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768397.675846:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768397.675888:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768397/real 1775768397] req@ffffa0e9d0540e00 x1862023861549824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768397.675904:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540e00 x1862023861549824/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768397.675913:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768397.675917:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768397.675921:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861549824:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768397.675943:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549952:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768397.675963:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768397.675974:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768397/real 1775768397] req@ffffa0e9d0543800 x1862023861549952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768397.675984:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543800 x1862023861549952/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768397.675990:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768397.675993:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768397.675996:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861549952:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768397.676001:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550080:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768397.676017:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768397.676026:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768397/real 1775768397] req@ffffa0e9d0540700 x1862023861550080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768397.676043:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540700 x1862023861550080/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768397.676051:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768397.676053:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768397.676056:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550080:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768398.251785:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768398.507636:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768399.275663:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768399.531637:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768400.299676:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768400.555627:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768401.323742:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768401.579626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768402.347739:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768402.603612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768402.795639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768402.795648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768402.795653:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768402.795663:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4131 00000100:00080000:0.0:1775768402.795675:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768402.795681:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768402.795719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861550208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768402.795749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768402.795752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768402.795753:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768402.795756:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4131 00000100:00080000:0.0:1775768402.795758:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768402.795760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768402.795768:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768402.795775:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861550336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768402.795805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768402.795810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768402.795812:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768402.795817:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4131 00000100:00080000:0.0:1775768402.795822:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768402.795826:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768402.795833:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768402.795843:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861550208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768402.795849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861550464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768402.795932:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768402.795979:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768402/real 1775768402] req@ffffa0e9d09ddf80 x1862023861550208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768402.796017:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861550208/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768402.796027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768402.796031:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768402.796035:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861550208:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768402.796043:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550336:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768402.796067:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768402.796079:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768402/real 1775768402] req@ffffa0e9d09dca80 x1862023861550336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768402.796091:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861550336/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768402.796098:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768402.796101:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768402.796103:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550336:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768402.796127:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550464:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768402.796145:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768402.796153:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768402/real 1775768402] req@ffffa0e9d09de680 x1862023861550464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768402.796160:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861550464/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768402.796173:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768402.796176:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768402.796179:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550464:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768403.371736:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768403.627821:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768404.395695:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768404.652385:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768405.419701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768405.675571:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768406.443710:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768406.699592:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768407.467674:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768407.723621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768407.915636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768407.915648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768407.915654:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768407.915665:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4136 00000100:00080000:0.0:1775768407.915677:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768407.915683:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768407.915723:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09de680 x1862023861550592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768407.915748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768407.915751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768407.915752:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768407.915754:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4136 00000100:00080000:0.0:1775768407.915756:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768407.915758:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768407.915781:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768407.915789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dca80 x1862023861550720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768407.915797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768407.915799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768407.915800:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768407.915804:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4136 00000100:00080000:0.0:1775768407.915806:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768407.915808:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768407.915811:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768407.915815:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861550592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768407.915822:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddf80 x1862023861550848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768407.915897:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768407.915952:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768407/real 1775768407] req@ffffa0e9d09de680 x1862023861550592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768407.915973:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09de680 x1862023861550592/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768407.915984:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768407.915988:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768407.915992:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861550592:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768407.916000:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550720:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768407.916022:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768407.916046:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768407/real 1775768407] req@ffffa0e9d09dca80 x1862023861550720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768407.916059:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dca80 x1862023861550720/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768407.916067:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768407.916070:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768407.916074:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550720:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768407.916105:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550848:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768407.916125:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768407.916135:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768407/real 1775768407] req@ffffa0e9d09ddf80 x1862023861550848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768407.916147:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddf80 x1862023861550848/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768407.916154:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768407.916158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768407.916161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861550848:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768408.491705:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768408.747616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768409.518416:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768409.771622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768410.539682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768410.795645:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768411.563713:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768411.819626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768412.587787:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768412.843597:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768413.035627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768413.035636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768413.035640:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768413.035649:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4141 00000100:00080000:2.0:1775768413.035657:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768413.035663:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768413.035697:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711180 x1862023861550976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768413.035722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768413.035726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768413.035728:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768413.035732:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4141 00000100:00080000:2.0:1775768413.035736:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768413.035740:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768413.035749:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768413.035773:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710380 x1862023861551104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768413.035780:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861550976:192.168.202.104@tcp:250:kworker.0 00000100:00080000:2.0:1775768413.035786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768413.035790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768413.035791:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768413.035814:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4141 00000100:00080000:2.0:1775768413.035818:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768413.035822:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768413.035827:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768413.035842:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0713b80 x1862023861551232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768413.035846:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768413.035904:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768413/real 1775768413] req@ffffa0e9d0711180 x1862023861550976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768413.035924:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711180 x1862023861550976/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768413.035933:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768413.035938:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768413.035941:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861550976:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768413.035965:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551104:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768413.035988:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768413.036000:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768413/real 1775768413] req@ffffa0e9d0710380 x1862023861551104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768413.036021:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710380 x1862023861551104/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768413.036029:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768413.036033:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768413.036036:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551104:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768413.036042:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551232:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768413.036062:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768413.036072:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768413/real 1775768413] req@ffffa0e9d0713b80 x1862023861551232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768413.036083:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0713b80 x1862023861551232/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768413.036091:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768413.036094:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768413.036097:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0713b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551232:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768413.611726:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768413.867613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768414.635641:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768414.891608:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768415.659632:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768415.915595:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768416.683710:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768416.939829:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768417.707701:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768417.963605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768418.155660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768418.155671:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768418.155676:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768418.155687:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4146 00000100:00080000:1.0:1775768418.155697:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768418.155704:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768418.155744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deea00 x1862023861551360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768418.155794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768418.155798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768418.155800:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768418.155804:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4146 00000100:00080000:1.0:1775768418.155808:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768418.155812:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768418.155823:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768418.155836:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded880 x1862023861551488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768418.155848:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768418.155850:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861551360:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768418.155852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768418.155855:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768418.155862:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4146 00000100:00080000:1.0:1775768418.155865:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768418.155869:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768418.155874:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768418.155885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee680 x1862023861551616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768418.155912:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768418.155947:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768418/real 1775768418] req@ffffa0e9c8deea00 x1862023861551360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768418.155962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deea00 x1862023861551360/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768418.155972:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768418.155976:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768418.155979:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861551360:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768418.156001:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551488:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768418.156041:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768418.156058:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768418/real 1775768418] req@ffffa0e9c8ded880 x1862023861551488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768418.156070:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded880 x1862023861551488/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768418.156078:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768418.156081:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768418.156084:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551488:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768418.156090:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551616:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768418.156126:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768418.156134:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768418/real 1775768418] req@ffffa0e9c8dee680 x1862023861551616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768418.156146:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee680 x1862023861551616/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768418.156154:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768418.156158:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768418.156161:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551616:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768418.731634:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768418.987614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768419.755727:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768420.011624:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768420.779726:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768421.035614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768421.803698:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768422.059586:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768422.827679:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768423.083570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768423.275613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768423.275624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768423.275641:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768423.275651:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4151 00000100:00080000:3.0:1775768423.275661:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768423.275665:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768423.275701:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0540a80 x1862023861551744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768423.275721:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768423.275724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768423.275725:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768423.275727:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4151 00000100:00080000:3.0:1775768423.275750:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768423.275753:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768423.275762:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768423.275773:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861551744:192.168.202.104@tcp:250:kworker.0 00000100:00100000:3.0:1775768423.275775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0542d80 x1862023861551872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768423.275787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768423.275789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768423.275791:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768423.275793:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4151 00000100:00080000:3.0:1775768423.275796:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768423.275798:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768423.275801:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768423.275808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541f80 x1862023861552000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768423.275867:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551872:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768423.275890:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552000:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768423.851791:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768424.107555:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768424.875679:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768425.131612:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768425.899690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768426.155614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000400:00000100:2.0:1775768426.347807:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.104:988 00000400:00000100:2.0:1775768426.347862:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.104@tcp at host 192.168.202.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768426.347878:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768426.347920:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000800:00000100:2.0:1775768426.347930:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.104@tcp 00000100:00100000:0.0:1775768426.348141:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768423/real 1775768426] req@ffffa0e9d0542d80 x1862023861551872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768426.348169:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0542d80 x1862023861551872/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768426.348182:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768426.348187:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768426.348191:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0542d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861551872:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768426.348199:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768423/real 1775768426] req@ffffa0e9d0541f80 x1862023861552000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768426.348209:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541f80 x1862023861552000/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768426.348216:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768426.348219:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768426.348222:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552000:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768426.348228:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768423/real 1775768426] req@ffffa0e9d0540a80 x1862023861551744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768426.348239:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0540a80 x1862023861551744/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768426.348246:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768426.348249:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768426.348252:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0540a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861551744:192.168.202.104@tcp:250:kworker.0 00010000:00010000:2.0:1775768426.923682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768427.179622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768427.947611:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768428.203626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768428.395633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768428.395645:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768428.395650:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768428.395660:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4156 00000100:00080000:1.0:1775768428.395687:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768428.395693:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768428.395726:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dedc00 x1862023861552128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768428.395754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768428.395757:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768428.395759:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768428.395763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4156 00000100:00080000:1.0:1775768428.395767:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768428.395771:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768428.395781:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768428.395794:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def480 x1862023861552256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768428.395799:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861552128:192.168.202.104@tcp:250:kworker.0 00000100:00080000:1.0:1775768428.395811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768428.395814:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768428.395816:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768428.395820:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4156 00000100:00080000:1.0:1775768428.395824:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768428.395828:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768428.395833:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768428.395845:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8def800 x1862023861552384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768428.395869:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768428.395909:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768428/real 1775768428] req@ffffa0e9c8dedc00 x1862023861552128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768428.395927:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dedc00 x1862023861552128/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768428.395937:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768428.395941:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768428.395944:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861552128:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768428.395968:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552256:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768428.395989:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768428.396001:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768428/real 1775768428] req@ffffa0e9c8def480 x1862023861552256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768428.396014:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def480 x1862023861552256/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768428.396022:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768428.396025:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768428.396028:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552256:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768428.396033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552384:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768428.396051:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768428.396060:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768428/real 1775768428] req@ffffa0e9c8def800 x1862023861552384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768428.396079:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8def800 x1862023861552384/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768428.396087:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768428.396089:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768428.396092:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552384:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768428.971751:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768429.227611:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768429.995684:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768430.251617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768431.020358:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768431.275614:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768432.043684:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768432.299633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768433.068443:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768433.323621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768433.515566:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768433.515598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768433.515603:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768433.515614:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4162 00000100:00080000:1.0:1775768433.515625:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768433.515632:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768433.515667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8deed80 x1862023861552512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768433.515696:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768433.515700:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768433.515702:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768433.515706:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4162 00000100:00080000:1.0:1775768433.515710:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768433.515713:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768433.515723:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768433.515735:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec380 x1862023861552640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768433.515744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768433.515747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768433.515749:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768433.515752:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4162 00000100:00080000:1.0:1775768433.515755:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768433.515758:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768433.515762:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768433.515770:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded500 x1862023861552768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768433.515799:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861552512:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775768433.515859:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768433.515897:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768433/real 1775768433] req@ffffa0e9c8deed80 x1862023861552512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768433.515915:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8deed80 x1862023861552512/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768433.515926:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768433.515930:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768433.515934:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861552512:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768433.515940:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552640:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768433.515967:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768433.515978:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768433/real 1775768433] req@ffffa0e9c8dec380 x1862023861552640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768433.515989:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec380 x1862023861552640/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768433.516008:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768433.516012:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768433.516015:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552640:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768433.516021:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552768:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768433.516038:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768433.516046:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768433/real 1775768433] req@ffffa0e9c8ded500 x1862023861552768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768433.516056:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded500 x1862023861552768/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768433.516064:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768433.516067:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768433.516070:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861552768:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768434.091621:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768434.347602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768435.115682:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768435.371638:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768436.139699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768436.395626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768437.163797:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768437.419662:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768438.187680:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768438.443616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768438.635653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768438.635664:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768438.635669:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768438.635678:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4167 00000100:00080000:3.0:1775768438.635689:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768438.635695:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768438.635733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861552896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768438.635764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768438.635784:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768438.635787:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768438.635791:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4167 00000100:00080000:3.0:1775768438.635797:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768438.635801:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768438.635811:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768438.635825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861553024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768438.635837:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861552896:192.168.202.104@tcp:250:kworker.0 00000100:00080000:3.0:1775768438.635844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768438.635848:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768438.635850:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768438.635856:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4167 00000100:00080000:3.0:1775768438.635859:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768438.635863:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768438.635868:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768438.635882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861553152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768438.635919:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768438.635958:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768438/real 1775768438] req@ffffa0e9d0541180 x1862023861552896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768438.635978:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861552896/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768438.635985:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768438.635989:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768438.636002:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861552896:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768438.636024:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553024:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768438.636039:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768438.636047:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768438/real 1775768438] req@ffffa0e9d0543480 x1862023861553024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768438.636054:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861553024/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768438.636059:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768438.636062:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768438.636063:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553024:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768438.636067:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553152:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768438.636076:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768438.636081:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768438/real 1775768438] req@ffffa0e9d0543100 x1862023861553152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768438.636087:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861553152/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768438.636092:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768438.636093:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768438.636095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553152:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768439.211699:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768439.467563:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768440.235761:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768440.491570:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768441.259690:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768441.515578:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768442.283788:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768442.539610:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768443.307767:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768443.563631:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768443.755638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768443.755648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768443.755672:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768443.755682:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4172 00000100:00080000:0.0:1775768443.755691:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768443.755697:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768443.755730:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09ddc00 x1862023861553280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768443.755758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768443.755762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768443.755763:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768443.755767:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4172 00000100:00080000:0.0:1775768443.755770:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768443.755773:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768443.755783:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768443.755794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd180 x1862023861553408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768443.755803:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861553280:192.168.202.104@tcp:250:kworker.0 00000100:00080000:0.0:1775768443.755805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768443.755810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768443.755812:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768443.755816:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4172 00000100:00080000:0.0:1775768443.755819:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768443.755822:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768443.755827:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768443.755838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd500 x1862023861553536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768443.755860:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768443.755895:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768443/real 1775768443] req@ffffa0e9d09ddc00 x1862023861553280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768443.755907:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09ddc00 x1862023861553280/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768443.755913:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768443.755916:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768443.755919:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861553280:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768443.755935:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553408:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768443.755946:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768443.755953:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768443/real 1775768443] req@ffffa0e9d09dd180 x1862023861553408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768443.755962:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd180 x1862023861553408/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768443.755969:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768443.755972:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768443.755975:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553408:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768443.755980:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553536:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768443.755997:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768443.756014:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768443/real 1775768443] req@ffffa0e9d09dd500 x1862023861553536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768443.756022:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd500 x1862023861553536/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768443.756027:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768443.756029:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768443.756031:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553536:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768444.331711:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768444.587811:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768445.355682:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768445.611613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768446.380707:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768446.635602:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768447.405693:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768447.659615:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768448.427702:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768448.683622:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768448.875633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768448.875645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768448.875650:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768448.875660:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4177 00000100:00080000:2.0:1775768448.875672:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768448.875679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768448.875723:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711c00 x1862023861553664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768448.875750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768448.875772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768448.875774:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768448.875779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4177 00000100:00080000:2.0:1775768448.875783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768448.875789:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768448.875790:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861553664:192.168.202.104@tcp:250:kworker.0 00000008:00100000:2.0:1775768448.875800:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768448.875815:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710000 x1862023861553792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768448.875826:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768448.875829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768448.875831:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768448.875834:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4177 00000100:00080000:2.0:1775768448.875840:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768448.875844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768448.875848:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768448.875859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0711f80 x1862023861553920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768448.875867:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768448.875912:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768448/real 1775768448] req@ffffa0e9d0711c00 x1862023861553664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768448.875930:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711c00 x1862023861553664/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768448.875940:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768448.875944:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768448.875948:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861553664:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768448.875973:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553792:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768448.875996:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768448.876007:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768448/real 1775768448] req@ffffa0e9d0710000 x1862023861553792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768448.876018:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710000 x1862023861553792/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768448.876038:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768448.876077:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768448.876081:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553792:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768448.876087:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553920:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768448.876110:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768448.876120:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768448/real 1775768448] req@ffffa0e9d0711f80 x1862023861553920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768448.876131:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0711f80 x1862023861553920/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768448.876138:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768448.876142:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768448.876144:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0711f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861553920:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768449.451672:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768449.707616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768450.475707:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768450.731661:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768451.499645:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768451.755593:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768452.526530:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768452.779640:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768453.547757:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768453.807960:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:1.0:1775768453.995898:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768453.995908:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768453.995913:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768453.995922:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4182 00000100:00080000:1.0:1775768453.995932:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768453.995939:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:1.0:1775768453.995983:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dee300 x1862023861554048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768453.996028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768453.996047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768453.996049:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768453.996054:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4182 00000100:00080000:1.0:1775768453.996059:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768453.996064:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768453.996074:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768453.996081:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861554048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768453.996095:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8dec000 x1862023861554176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768453.996105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768453.996109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768453.996111:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768453.996115:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4182 00000100:00080000:1.0:1775768453.996133:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768453.996136:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:1.0:1775768453.996141:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768453.996144:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768453.996153:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9c8ded180 x1862023861554304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768453.996187:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768453/real 1775768453] req@ffffa0e9c8dee300 x1862023861554048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768453.996204:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dee300 x1862023861554048/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768453.996215:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768453.996219:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768453.996222:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861554048:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768453.996245:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554176:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768453.996264:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768453.996274:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768453/real 1775768453] req@ffffa0e9c8dec000 x1862023861554176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768453.996285:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8dec000 x1862023861554176/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768453.996293:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768453.996296:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768453.996299:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554176:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768453.996304:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554304:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768453.996318:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768453.996334:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768453/real 1775768453] req@ffffa0e9c8ded180 x1862023861554304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768453.996345:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9c8ded180 x1862023861554304/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768453.996353:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768453.996356:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768453.996420:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9c8ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554304:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768454.571640:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768454.827605:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768455.595683:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768455.851616:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768456.619739:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768456.875754:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768457.643743:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768457.899626:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768458.671560:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768458.925056:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:2.0:1775768459.115641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768459.115653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768459.115657:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768459.115668:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4187 00000100:00080000:2.0:1775768459.115679:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768459.115686:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:2.0:1775768459.115726:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0712d80 x1862023861554432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768459.115761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768459.115765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768459.115767:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768459.115771:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4187 00000100:00080000:2.0:1775768459.115775:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768459.115778:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768459.115788:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768459.115821:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710a80 x1862023861554560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768459.115831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768459.115833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768459.115835:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768459.115839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4187 00000100:00080000:2.0:1775768459.115843:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768459.115847:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:2.0:1775768459.115852:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768459.115861:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0710700 x1862023861554688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768459.115970:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861554432:192.168.202.104@tcp:250:kworker.0 00000800:00000100:0.0:1775768459.116032:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768459.116087:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768459/real 1775768459] req@ffffa0e9d0712d80 x1862023861554432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768459.116106:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0712d80 x1862023861554432/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768459.116117:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768459.116121:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768459.116124:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0712d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861554432:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768459.116131:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554560:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768459.116154:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768459.116176:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768459/real 1775768459] req@ffffa0e9d0710a80 x1862023861554560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768459.116188:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710a80 x1862023861554560/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768459.116208:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768459.116211:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768459.116215:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554560:192.168.202.104@tcp:38:kworker.0 00000100:00100000:0.0:1775768459.116221:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554688:192.168.202.104@tcp:38:kworker.0 00000800:00000100:0.0:1775768459.116248:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:0.0:1775768459.116258:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768459/real 1775768459] req@ffffa0e9d0710700 x1862023861554688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768459.116269:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0710700 x1862023861554688/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768459.116288:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768459.116292:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:0.0:1775768459.116295:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0710700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554688:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768459.691716:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768459.948015:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768460.715672:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768460.971633:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768461.742145:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768461.995639:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768462.764472:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768463.019613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768463.787692:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768464.043617:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768464.235642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768464.235654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768464.235659:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768464.235669:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4192 00000100:00080000:0.0:1775768464.235680:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768464.235687:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:0.0:1775768464.235727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09df800 x1862023861554816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768464.235771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768464.235790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768464.235792:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768464.235796:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4192 00000100:00080000:0.0:1775768464.235801:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768464.235806:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768464.235816:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768464.235827:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861554816:192.168.202.104@tcp:250:kworker.0 00000100:00100000:0.0:1775768464.235831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dc700 x1862023861554944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768464.235843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768464.235846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768464.235847:0:37:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768464.235851:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4192 00000100:00080000:0.0:1775768464.235855:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768464.235858:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:0.0:1775768464.235863:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768464.235873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d09dd880 x1862023861555072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768464.235906:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768464.235960:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768464/real 1775768464] req@ffffa0e9d09df800 x1862023861554816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768464.235980:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09df800 x1862023861554816/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768464.236002:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768464.236006:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768464.236010:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861554816:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768464.236033:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554944:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768464.236057:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768464.236069:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768464/real 1775768464] req@ffffa0e9d09dc700 x1862023861554944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768464.236082:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dc700 x1862023861554944/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768464.236089:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768464.236092:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768464.236095:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861554944:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768464.236101:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861555072:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768464.236120:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768464.236131:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768464/real 1775768464] req@ffffa0e9d09dd880 x1862023861555072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768464.236142:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d09dd880 x1862023861555072/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768464.236149:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768464.236153:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768464.236156:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d09dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861555072:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768464.811774:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768465.067850:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768465.835862:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768466.091704:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768466.859928:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768467.115613:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768467.883664:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768468.139747:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768468.910060:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768469.163738:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768469.355745:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2bf32c54-7e4b-4438-8419-d956344e0e00->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768469.355758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768469.355829:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768469.355840:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.104@tcp: connect to NID 192.168.202.104@tcp last attempt 4197 00000100:00080000:3.0:1775768469.355861:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768469.355868:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.104@tcp: import ffffa0e9d09a7800 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000100:00100000:3.0:1775768469.355913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543100 x1862023861555200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768469.355962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768469.355966:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768469.355967:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768469.355971:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4197 00000100:00080000:3.0:1775768469.355975:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768469.355979:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffffa0e9c80cb000: import ffffa0e9d0199000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768469.355987:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768469.355999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0543480 x1862023861555328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768469.356008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6a4854d3-5a79-4a54-82b7-484d046ae78a->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768469.356011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768469.356013:0:8:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768469.356017:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: connect to NID 192.168.202.104@tcp last attempt 4197 00000100:00080000:3.0:1775768469.356020:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768469.356022:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffffa0e9c80cb000: import ffffa0e9d04cb000 using connection 192.168.202.104@tcp/192.168.202.104@tcp 00000008:00100000:3.0:1775768469.356026:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768469.356034:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0e9d0541180 x1862023861555456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768469.356088:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861555200:192.168.202.104@tcp:250:kworker.0 00000800:00000100:1.0:1775768469.356156:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768469.356209:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768469/real 1775768469] req@ffffa0e9d0543100 x1862023861555200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768469.356227:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543100 x1862023861555200/t0(0) o250->MGC192.168.202.104@tcp@192.168.202.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768469.356237:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d09a7800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768469.356241:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768469.356246:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2bf32c54-7e4b-4438-8419-d956344e0e00:2400:1862023861555200:192.168.202.104@tcp:250:kworker.0 00000100:00100000:1.0:1775768469.356253:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861555328:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768469.356276:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768469.356288:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768469/real 1775768469] req@ffffa0e9d0543480 x1862023861555328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768469.356299:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0543480 x1862023861555328/t0(0) o38->lustre-MDT0000-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768469.356308:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d0199000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768469.356311:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768469.356314:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0543480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861555328:192.168.202.104@tcp:38:kworker.0 00000100:00100000:1.0:1775768469.356320:0:2400:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861555456:192.168.202.104@tcp:38:kworker.0 00000800:00000100:1.0:1775768469.356338:0:2400:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.104@tcp 00000100:00100000:1.0:1775768469.356348:0:2400:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768469/real 1775768469] req@ffffa0e9d0541180 x1862023861555456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768469.356380:0:2400:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0e9d0541180 x1862023861555456/t0(0) o38->lustre-MDT0001-mdc-ffffa0e9c80cb000@192.168.202.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768469.356392:0:2400:0:(import.c:65:import_set_state_nolock()) ffffa0e9d04cb000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768469.356395:0:2400:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.104@tcp failed (-110) 00000100:00100000:1.0:1775768469.356398:0:2400:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffffa0e9d0541180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6a4854d3-5a79-4a54-82b7-484d046ae78a:2400:1862023861555456:192.168.202.104@tcp:38:kworker.0 00010000:00010000:2.0:1775768469.931956:0:6650:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768470.187621:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768470.956802:0:6172:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768471.211628:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds 00010000:00010000:2.0:1775768471.981251:0:6171:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768472.235967:0:14656:0:(client.c:2691:ptlrpc_set_wait()) set ffffa0e9d09d6b40 going to sleep for 0 seconds Debug log: 47630 lines, 47630 kept, 0 dropped, 0 bad.